diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..92dbe2fb9bb65a171c7546499c8712b969ba8b93 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,30 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/1911fd281a78caf78c1d504c331f793b filter=lfs diff=lfs merge=lfs -text +owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/fcb7dd856b448b10dbef9888b62a0e86 filter=lfs diff=lfs merge=lfs -text +owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/0d83f591b4a942394ede303a3e0f7014 filter=lfs diff=lfs merge=lfs -text +owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/f025991dbea51509b0c6be19d75fcce7 filter=lfs diff=lfs merge=lfs -text +owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/ac0a06c0467678e073f7c8f152212fc4 filter=lfs diff=lfs merge=lfs -text +owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/b9f1f59ddecdf266b47b1febcb410b08 filter=lfs diff=lfs merge=lfs -text +owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/c0efe874dd28f5c61737aebcb57da304 filter=lfs diff=lfs merge=lfs -text +owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/2ddddee6a811084c38a9e8458b0ced0d filter=lfs diff=lfs merge=lfs -text +owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/78dc8adeb1f9e442151cca50674329e6 filter=lfs diff=lfs merge=lfs -text +owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/9db3aae2c824f358c75bbbdf13fac62d filter=lfs diff=lfs merge=lfs -text +text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/b6d45a6e08a3adfce23fe173b50b483a filter=lfs diff=lfs merge=lfs -text +text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/c14ea4fd91c09ebb390aa17edb2e647b filter=lfs diff=lfs merge=lfs -text +text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/fba552e9dfbcec2f5571cafbf0a95d35 filter=lfs diff=lfs merge=lfs -text +text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/038873e5c4f3aff476bce1260152c3b3 filter=lfs diff=lfs merge=lfs -text +text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/0511ed773775dd2efbe8f7ad95ecfdd5 filter=lfs diff=lfs merge=lfs -text +text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/31fbd64151e1c27957b25b54f68eeddf filter=lfs diff=lfs merge=lfs -text +text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/764aa799191684deed3790948a34d993 filter=lfs diff=lfs merge=lfs -text +text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/9992e06dbbc03954ae4aa2764c028315 filter=lfs diff=lfs merge=lfs -text +tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/065ec2be3d26fef4dbbc9a9050f17a71 filter=lfs diff=lfs merge=lfs -text +tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/3274ff5e02e402c544b9a447121b27f4 filter=lfs diff=lfs merge=lfs -text +tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/9f27210eedd8af7596fe78838a126b12 filter=lfs diff=lfs merge=lfs -text +tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/15f1356815fabf8024fca8cee55adc78 filter=lfs diff=lfs merge=lfs -text +tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/68757985fb3b0f9dbc7d2a51d124fd5b filter=lfs diff=lfs merge=lfs -text +tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/a4fbec2e52c1c7bf2d2e05d2a90ebae6 filter=lfs diff=lfs merge=lfs -text +tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/c9d5308de0df3c863e62db6465227bed filter=lfs diff=lfs merge=lfs -text +tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/ea92c267744755209c1853785346aad1 filter=lfs diff=lfs merge=lfs -text +tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/f8169566ee0804773f27cf30054bc094 filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8600373cdfb2ee2c3bfafdbe2b39a35177a49493 --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +--- +license: apache-2.0 +tags: +- discrete-diffusion +- jax +- flax +--- + +# Binary Discrete Diffusion: text models + +The text models of [Binary Discrete Diffusion: Candidate-Set Diffusion for Logic and Planning +Tasks](https://arxiv.org/abs/2610.28543): one diffusion transformer per corpus and noise kernel (`masked`, +`uniform`, `binary_symmetric`, `binary_absorbing`). Each `//` is a run directory of +[binary-discrete-diffusion](https://github.com/rodcasnog/binary-discrete-diffusion), trained with +`configs/paper//.yaml`: the composed config in `.hydra/config.yaml` and the EMA parameters of +the final step in `checkpoints_ema//`. + +| Corpus | Directory | Checkpoint | Tokenizer | +|---|---|---|---| +| Text8 | `text8/` | `checkpoints_ema/1000000` | characters, written by `scripts/prep_text8.py` | +| TinyStories | `tinystories/` | `checkpoints_ema/500000` | [rodcasnog/tinystories-bpe-1k](https://huggingface.co/rodcasnog/tinystories-bpe-1k) | +| OpenWebText | `owt/` | `checkpoints_ema/250000` | [dvruette/openwebtext-bpe-1k](https://huggingface.co/dvruette/openwebtext-bpe-1k) | + +## Usage + +From a clone of the code repository, after `uv sync`: + +```bash +uv run hf download rodcasnog/binary-discrete-diffusion --include "text8/binary_symmetric/*" --local-dir outputs/hf +uv run python -m binary_diffusion.sample checkpoint_dir=outputs/hf/text8/binary_symmetric/checkpoints_ema/1000000 \ + sampler.nfe=256 sampler.spacing=log sampler.eta=1 sampler.lam=1 out=outputs/text8-bsym/samples +``` + +Text8 models read the tokenizer in `data/text8`, TinyStories samples take their lengths from `data/tinystories`, +and NELBO and MAUVE read the corpus; the code's README lists the data and evaluation commands. + +## License + +Apache-2.0. + +## Citation + +```bibtex +@misc{casadonoguerales2026binarydiffusion, + title = {Binary Discrete Diffusion: Candidate-Set Diffusion for + Logic and Planning Tasks}, + author = {Casado Noguerales, Rodrigo and von R{\"u}tte, Dimitri + and Sch{\"o}lkopf, Bernhard and Hofmann, Thomas + and Raoufi, Aran}, + year = {2026}, + eprint = {2610.28543}, + archivePrefix = {arXiv}, + primaryClass = {cs.LG}, + url = {https://arxiv.org/abs/2610.28543}, +} +``` diff --git a/owt/binary_absorbing/.hydra/config.yaml b/owt/binary_absorbing/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1621b0fd7dbfe98540099127f138b725f080aa30 --- /dev/null +++ b/owt/binary_absorbing/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: binary_absorbing + objective: elbo + schedule: log + t_min: 0.0005 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 1024 + num_layers: 16 + num_heads: 16 + mlp_dim: 4096 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 250000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: false + dataset_path: data/owt + tokenizer_name: dvruette/openwebtext-bpe-1k + max_seq_len: 1024 diff --git a/owt/binary_absorbing/checkpoints_ema/250000/_CHECKPOINT_METADATA b/owt/binary_absorbing/checkpoints_ema/250000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..519bd20bac214665bd6f922dc7441328fb89c8fb --- /dev/null +++ b/owt/binary_absorbing/checkpoints_ema/250000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1790153271206075069, "commit_timestamp_nsecs": 1790153325607377156, "custom_metadata": {}} \ No newline at end of file diff --git a/owt/binary_absorbing/checkpoints_ema/250000/state/_METADATA b/owt/binary_absorbing/checkpoints_ema/250000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..37311c64ac7d4ec3c5ae02063e0c21c8d7c54b16 --- /dev/null +++ b/owt/binary_absorbing/checkpoints_ema/250000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '12', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '12', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '12', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '12', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '12', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '12', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '12', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '13', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '13', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '13', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '13', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '13', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '13', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '13', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '14', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '14', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '14', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '14', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '14', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '14', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '14', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '15', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '15', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '15', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '15', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '15', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '15', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '15', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1025, 512]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1024]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 2048]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 1024]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/owt/binary_absorbing/checkpoints_ema/250000/state/_sharding b/owt/binary_absorbing/checkpoints_ema/250000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..6e29cd57f5b8b606450322f4cbef3a7b2a04dc6e --- /dev/null +++ b/owt/binary_absorbing/checkpoints_ema/250000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}"} \ No newline at end of file diff --git a/owt/binary_absorbing/checkpoints_ema/250000/state/array_metadatas/process_0 b/owt/binary_absorbing/checkpoints_ema/250000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..9864c53cd5977c170b473611c018797d038a7c41 --- /dev/null +++ b/owt/binary_absorbing/checkpoints_ema/250000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [64], "chunk_shape": [64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [64], "chunk_shape": [64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [64, 128], "chunk_shape": [64, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [1025, 512], "chunk_shape": [1025, 512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1024], "chunk_shape": [1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [64, 2048], "chunk_shape": [64, 2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [512, 1024], "chunk_shape": [512, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}]} \ No newline at end of file diff --git a/owt/binary_absorbing/checkpoints_ema/250000/state/d/8592ee904b52fdb8ef217541e7b80dfe b/owt/binary_absorbing/checkpoints_ema/250000/state/d/8592ee904b52fdb8ef217541e7b80dfe new file mode 100644 index 0000000000000000000000000000000000000000..4954b7723f1958b28267a91ed9baafeeaa43c289 Binary files /dev/null and b/owt/binary_absorbing/checkpoints_ema/250000/state/d/8592ee904b52fdb8ef217541e7b80dfe differ diff --git a/owt/binary_absorbing/checkpoints_ema/250000/state/manifest.ocdbt b/owt/binary_absorbing/checkpoints_ema/250000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..5df179643b91eaae9322f5b2f61439f58a74f0ee Binary files /dev/null and b/owt/binary_absorbing/checkpoints_ema/250000/state/manifest.ocdbt differ diff --git a/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/1911fd281a78caf78c1d504c331f793b b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/1911fd281a78caf78c1d504c331f793b new file mode 100644 index 0000000000000000000000000000000000000000..8e176860121eb945c2efb475c9c9cc97637bc105 --- /dev/null +++ b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/1911fd281a78caf78c1d504c331f793b @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e4a6dd0002d87eb8bf4b830a9943bd2e6ba741eb318c6eca626a4cd4ae98a11 +size 702197760 diff --git a/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/27676e82037cedd572911bc794a47eb1 b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/27676e82037cedd572911bc794a47eb1 new file mode 100644 index 0000000000000000000000000000000000000000..c40c4950104d5a206a3f7e1137352a3cefab660a Binary files /dev/null and b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/27676e82037cedd572911bc794a47eb1 differ diff --git a/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/98c08ebf57454a777c9cf2711d0c0cc3 b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/98c08ebf57454a777c9cf2711d0c0cc3 new file mode 100644 index 0000000000000000000000000000000000000000..34a69e0717b9ff772949069f73b1e3d06f2bb0ed Binary files /dev/null and b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/98c08ebf57454a777c9cf2711d0c0cc3 differ diff --git a/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/9f83d537925bd13f3da3be0ee12981db b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/9f83d537925bd13f3da3be0ee12981db new file mode 100644 index 0000000000000000000000000000000000000000..4b2775ca29ae2c193c0662ef9dca0f068c6012ea Binary files /dev/null and b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/9f83d537925bd13f3da3be0ee12981db differ diff --git a/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/fcb7dd856b448b10dbef9888b62a0e86 b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/fcb7dd856b448b10dbef9888b62a0e86 new file mode 100644 index 0000000000000000000000000000000000000000..2da11271ef52d4184461f963a90fcf5d048190f4 --- /dev/null +++ b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/d/fcb7dd856b448b10dbef9888b62a0e86 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80927cceaeeaa07cbfc7fa105bf0a96fb10e6072e022276ff3fed1647c0051bf +size 101085184 diff --git a/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..1c1ec0116e29880977df0f4b0c2e1c005d19794e Binary files /dev/null and b/owt/binary_absorbing/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt differ diff --git a/owt/binary_symmetric/.hydra/config.yaml b/owt/binary_symmetric/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9eda1155582fc88c5bf1cd97de339458d3b95332 --- /dev/null +++ b/owt/binary_symmetric/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: binary_symmetric + objective: elbo + schedule: log + t_min: 0.0005 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 1024 + num_layers: 16 + num_heads: 16 + mlp_dim: 4096 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 250000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: false + dataset_path: data/owt + tokenizer_name: dvruette/openwebtext-bpe-1k + max_seq_len: 1024 diff --git a/owt/binary_symmetric/checkpoints_ema/250000/_CHECKPOINT_METADATA b/owt/binary_symmetric/checkpoints_ema/250000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..5ebc3ea44f777254dd02f6285f2b01f7338c35e2 --- /dev/null +++ b/owt/binary_symmetric/checkpoints_ema/250000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789856854938633902, "commit_timestamp_nsecs": 1789856875216252659, "custom_metadata": {}} \ No newline at end of file diff --git a/owt/binary_symmetric/checkpoints_ema/250000/state/_METADATA b/owt/binary_symmetric/checkpoints_ema/250000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..37311c64ac7d4ec3c5ae02063e0c21c8d7c54b16 --- /dev/null +++ b/owt/binary_symmetric/checkpoints_ema/250000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '12', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '12', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '12', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '12', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '12', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '12', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '12', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '13', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '13', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '13', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '13', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '13', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '13', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '13', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '14', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '14', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '14', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '14', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '14', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '14', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '14', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '15', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '15', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '15', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '15', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '15', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '15', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '15', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1025, 512]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1024]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 2048]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 1024]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/owt/binary_symmetric/checkpoints_ema/250000/state/_sharding b/owt/binary_symmetric/checkpoints_ema/250000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..6e29cd57f5b8b606450322f4cbef3a7b2a04dc6e --- /dev/null +++ b/owt/binary_symmetric/checkpoints_ema/250000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}"} \ No newline at end of file diff --git a/owt/binary_symmetric/checkpoints_ema/250000/state/array_metadatas/process_0 b/owt/binary_symmetric/checkpoints_ema/250000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..9864c53cd5977c170b473611c018797d038a7c41 --- /dev/null +++ b/owt/binary_symmetric/checkpoints_ema/250000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [64], "chunk_shape": [64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [64], "chunk_shape": [64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [64, 128], "chunk_shape": [64, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [1025, 512], "chunk_shape": [1025, 512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1024], "chunk_shape": [1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [64, 2048], "chunk_shape": [64, 2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [512, 1024], "chunk_shape": [512, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}]} \ No newline at end of file diff --git a/owt/binary_symmetric/checkpoints_ema/250000/state/d/55cf5f4005eed08f8ba1040d27138e3a b/owt/binary_symmetric/checkpoints_ema/250000/state/d/55cf5f4005eed08f8ba1040d27138e3a new file mode 100644 index 0000000000000000000000000000000000000000..73fdf1782495768db0bfe2b54e22c13a56bf6e25 Binary files /dev/null and b/owt/binary_symmetric/checkpoints_ema/250000/state/d/55cf5f4005eed08f8ba1040d27138e3a differ diff --git a/owt/binary_symmetric/checkpoints_ema/250000/state/manifest.ocdbt b/owt/binary_symmetric/checkpoints_ema/250000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..d30f07c65f3ef6c22189dd944dc5ef97c7385763 Binary files /dev/null and b/owt/binary_symmetric/checkpoints_ema/250000/state/manifest.ocdbt differ diff --git a/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/0d83f591b4a942394ede303a3e0f7014 b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/0d83f591b4a942394ede303a3e0f7014 new file mode 100644 index 0000000000000000000000000000000000000000..0748a5c033daa9b9ea36d350754c4780c8c18703 --- /dev/null +++ b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/0d83f591b4a942394ede303a3e0f7014 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cdd20dc05da3945e84b7fd69823b5d4076aa3c06299530dd64371df982c7a6d +size 61767680 diff --git a/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/231b3b6e0fc19f8395da6234274258ed b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/231b3b6e0fc19f8395da6234274258ed new file mode 100644 index 0000000000000000000000000000000000000000..a3465f4e337d0165eaf07783cbe532bf871be3f0 Binary files /dev/null and b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/231b3b6e0fc19f8395da6234274258ed differ diff --git a/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/a167e24b935d5894f6546cab26088c63 b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/a167e24b935d5894f6546cab26088c63 new file mode 100644 index 0000000000000000000000000000000000000000..c5d4c3e7dc9b92b6087a5b85c1e1cc1a6ad0d75f Binary files /dev/null and b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/a167e24b935d5894f6546cab26088c63 differ diff --git a/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/e5fe6da64031d80960f6c31af4121792 b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/e5fe6da64031d80960f6c31af4121792 new file mode 100644 index 0000000000000000000000000000000000000000..c54ce2773babe925f3eee7193321816f190182ff Binary files /dev/null and b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/e5fe6da64031d80960f6c31af4121792 differ diff --git a/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/f025991dbea51509b0c6be19d75fcce7 b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/f025991dbea51509b0c6be19d75fcce7 new file mode 100644 index 0000000000000000000000000000000000000000..2afc5c13898ebb5029cfda9abd5ded2e61b01b02 --- /dev/null +++ b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/d/f025991dbea51509b0c6be19d75fcce7 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2751feb69c9698d76d5c61a87eee7280eb448b77555c6ee95413f1eec467e924 +size 741527552 diff --git a/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..b4f9bd39e5dcae4fda606082e1dd7479e8d8b401 Binary files /dev/null and b/owt/binary_symmetric/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt differ diff --git a/owt/masked/.hydra/config.yaml b/owt/masked/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30a222a7d4e4a8679c0411b99647cd90b6ff1de1 --- /dev/null +++ b/owt/masked/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: masked + objective: elbo + schedule: linear + t_min: 0.0005 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 1024 + num_layers: 16 + num_heads: 16 + mlp_dim: 4096 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 250000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: false + dataset_path: data/owt + tokenizer_name: dvruette/openwebtext-bpe-1k + max_seq_len: 1024 diff --git a/owt/masked/checkpoints_ema/250000/_CHECKPOINT_METADATA b/owt/masked/checkpoints_ema/250000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..e4dad173f7f1361a0f738496cff5801d51bfa9f6 --- /dev/null +++ b/owt/masked/checkpoints_ema/250000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789848731736373796, "commit_timestamp_nsecs": 1789848746077450748, "custom_metadata": {}} \ No newline at end of file diff --git a/owt/masked/checkpoints_ema/250000/state/_METADATA b/owt/masked/checkpoints_ema/250000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..37311c64ac7d4ec3c5ae02063e0c21c8d7c54b16 --- /dev/null +++ b/owt/masked/checkpoints_ema/250000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '12', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '12', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '12', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '12', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '12', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '12', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '12', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '13', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '13', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '13', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '13', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '13', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '13', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '13', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '14', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '14', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '14', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '14', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '14', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '14', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '14', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '15', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '15', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '15', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '15', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '15', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '15', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '15', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1025, 512]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1024]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 2048]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 1024]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/owt/masked/checkpoints_ema/250000/state/_sharding b/owt/masked/checkpoints_ema/250000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..6e29cd57f5b8b606450322f4cbef3a7b2a04dc6e --- /dev/null +++ b/owt/masked/checkpoints_ema/250000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}"} \ No newline at end of file diff --git a/owt/masked/checkpoints_ema/250000/state/array_metadatas/process_0 b/owt/masked/checkpoints_ema/250000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..9864c53cd5977c170b473611c018797d038a7c41 --- /dev/null +++ b/owt/masked/checkpoints_ema/250000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [64], "chunk_shape": [64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [64], "chunk_shape": [64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [64, 128], "chunk_shape": [64, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [1025, 512], "chunk_shape": [1025, 512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1024], "chunk_shape": [1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [64, 2048], "chunk_shape": [64, 2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [512, 1024], "chunk_shape": [512, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}]} \ No newline at end of file diff --git a/owt/masked/checkpoints_ema/250000/state/d/2acdebb668f3bc0d82e8744662d2b0b8 b/owt/masked/checkpoints_ema/250000/state/d/2acdebb668f3bc0d82e8744662d2b0b8 new file mode 100644 index 0000000000000000000000000000000000000000..d4a8bc0ed232dafe2c13c477e0d73b1f5604f3ab Binary files /dev/null and b/owt/masked/checkpoints_ema/250000/state/d/2acdebb668f3bc0d82e8744662d2b0b8 differ diff --git a/owt/masked/checkpoints_ema/250000/state/manifest.ocdbt b/owt/masked/checkpoints_ema/250000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..ea65e015fb3540ea62c5e9c744c16e567077ef0c Binary files /dev/null and b/owt/masked/checkpoints_ema/250000/state/manifest.ocdbt differ diff --git a/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/508f7ea036dd34bce5216c14c7491a4c b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/508f7ea036dd34bce5216c14c7491a4c new file mode 100644 index 0000000000000000000000000000000000000000..223351377228c6b1937090221348dbb7035c6d0e Binary files /dev/null and b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/508f7ea036dd34bce5216c14c7491a4c differ diff --git a/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/7115c12b5f3d9527ae218826e6092360 b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/7115c12b5f3d9527ae218826e6092360 new file mode 100644 index 0000000000000000000000000000000000000000..c40c4950104d5a206a3f7e1137352a3cefab660a Binary files /dev/null and b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/7115c12b5f3d9527ae218826e6092360 differ diff --git a/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/ac0a06c0467678e073f7c8f152212fc4 b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/ac0a06c0467678e073f7c8f152212fc4 new file mode 100644 index 0000000000000000000000000000000000000000..5943d3e4b3905b88f5177b16e5678100fb0ba9ea --- /dev/null +++ b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/ac0a06c0467678e073f7c8f152212fc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5807857b1078954d6a38da05ab6f99194ebedb2fcdc93775ff726499f2c69105 +size 11698176 diff --git a/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/b9f1f59ddecdf266b47b1febcb410b08 b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/b9f1f59ddecdf266b47b1febcb410b08 new file mode 100644 index 0000000000000000000000000000000000000000..a2780e77b1098b7ea83bf94f8d5422e8f6dacbef --- /dev/null +++ b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/b9f1f59ddecdf266b47b1febcb410b08 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daad52d734c00062f13d2e89c41e4123c703cd4c1dd7509e6640d1683d639bad +size 746074112 diff --git a/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/c0efe874dd28f5c61737aebcb57da304 b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/c0efe874dd28f5c61737aebcb57da304 new file mode 100644 index 0000000000000000000000000000000000000000..a2e51108d340338c3738d5b3a80d0604ccd104d3 --- /dev/null +++ b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/d/c0efe874dd28f5c61737aebcb57da304 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c22eafb23c0f2bad7d36e42ede2b3ae9c582f00c202e1fbd8b4642c751f415 +size 44679168 diff --git a/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..023b97366e44d5cda59531e99708df26b5ebcab9 Binary files /dev/null and b/owt/masked/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt differ diff --git a/owt/uniform/.hydra/config.yaml b/owt/uniform/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..329ee44a3b426594dc2b8e6da0dca4f6034d650e --- /dev/null +++ b/owt/uniform/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: uniform + objective: elbo + schedule: linear + t_min: 0.0005 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 1024 + num_layers: 16 + num_heads: 16 + mlp_dim: 4096 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 250000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: false + dataset_path: data/owt + tokenizer_name: dvruette/openwebtext-bpe-1k + max_seq_len: 1024 diff --git a/owt/uniform/checkpoints_ema/250000/_CHECKPOINT_METADATA b/owt/uniform/checkpoints_ema/250000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..498396814ef86f42222b8cf2592a70fe6cd3b256 --- /dev/null +++ b/owt/uniform/checkpoints_ema/250000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789850806155219956, "commit_timestamp_nsecs": 1789850870196851155, "custom_metadata": {}} \ No newline at end of file diff --git a/owt/uniform/checkpoints_ema/250000/state/_METADATA b/owt/uniform/checkpoints_ema/250000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..37311c64ac7d4ec3c5ae02063e0c21c8d7c54b16 --- /dev/null +++ b/owt/uniform/checkpoints_ema/250000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '12', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '12', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '12', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '12', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '12', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '12', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '12', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '12', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "12", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '13', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '13', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '13', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '13', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '13', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '13', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '13', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '13', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "13", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '14', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '14', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '14', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '14', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '14', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '14', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '14', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '14', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "14", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '15', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 6144]}}, "('params', 'blocks', '15', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [8, 64, 1024]}}, "('params', 'blocks', '15', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 16, 3, 64]}}, "('params', 'blocks', '15', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048]}}, "('params', 'blocks', '15', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 4096]}}, "('params', 'blocks', '15', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [2048, 1024]}}, "('params', 'blocks', '15', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'blocks', '15', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "15", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1025, 512]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1024]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [64, 2048]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512, 1024]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [512]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/owt/uniform/checkpoints_ema/250000/state/_sharding b/owt/uniform/checkpoints_ema/250000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..6e29cd57f5b8b606450322f4cbef3a7b2a04dc6e --- /dev/null +++ b/owt/uniform/checkpoints_ema/250000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMi5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xMy5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4xNS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [2], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}, {\"id\": 1}]}}"} \ No newline at end of file diff --git a/owt/uniform/checkpoints_ema/250000/state/array_metadatas/process_0 b/owt/uniform/checkpoints_ema/250000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..9864c53cd5977c170b473611c018797d038a7c41 --- /dev/null +++ b/owt/uniform/checkpoints_ema/250000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.12.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.13.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.14.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.adaLN_modulation.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.adaLN_modulation.kernel.value", "write_shape": [64, 6144], "chunk_shape": [64, 6144], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.attn_out.value", "write_shape": [8, 64, 1024], "chunk_shape": [8, 64, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.attn_qkv.value", "write_shape": [512, 16, 3, 64], "chunk_shape": [512, 16, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc1.bias.value", "write_shape": [2048], "chunk_shape": [2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc1.kernel.value", "write_shape": [512, 4096], "chunk_shape": [512, 4096], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc2.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.mlp_fc2.kernel.value", "write_shape": [2048, 1024], "chunk_shape": [2048, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.norm1.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.15.norm2.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [64], "chunk_shape": [64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [64], "chunk_shape": [64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [64, 128], "chunk_shape": [64, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [1025, 512], "chunk_shape": [1025, 512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1024], "chunk_shape": [1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [64, 2048], "chunk_shape": [64, 2048], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [512, 1024], "chunk_shape": [512, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [512], "chunk_shape": [512], "ext_metadata": null}}]} \ No newline at end of file diff --git a/owt/uniform/checkpoints_ema/250000/state/d/f366cf0b7835063df6a58948d3a0cb5c b/owt/uniform/checkpoints_ema/250000/state/d/f366cf0b7835063df6a58948d3a0cb5c new file mode 100644 index 0000000000000000000000000000000000000000..63d3f70abdd15aaca2ab2dbd419442b1c5c08a6a Binary files /dev/null and b/owt/uniform/checkpoints_ema/250000/state/d/f366cf0b7835063df6a58948d3a0cb5c differ diff --git a/owt/uniform/checkpoints_ema/250000/state/manifest.ocdbt b/owt/uniform/checkpoints_ema/250000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..6af43394fc3770a1c51087e5b0dda73783b7bb6c Binary files /dev/null and b/owt/uniform/checkpoints_ema/250000/state/manifest.ocdbt differ diff --git a/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/2ddddee6a811084c38a9e8458b0ced0d b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/2ddddee6a811084c38a9e8458b0ced0d new file mode 100644 index 0000000000000000000000000000000000000000..4f6b47efb9b6355e65438ff473e892b1a871ab14 --- /dev/null +++ b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/2ddddee6a811084c38a9e8458b0ced0d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9134fa5f0bc398d9d936f86922352e48f05667ef033eec408cf688eccac350b4 +size 746369024 diff --git a/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/78dc8adeb1f9e442151cca50674329e6 b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/78dc8adeb1f9e442151cca50674329e6 new file mode 100644 index 0000000000000000000000000000000000000000..efb46c903be85e07bea815ef89e87ec5d5d6defe --- /dev/null +++ b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/78dc8adeb1f9e442151cca50674329e6 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3d1a50eea28d7c0d622cf5e27de2d8786cb67aaade8a614c3b77b0a5703a361 +size 52453376 diff --git a/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/834911912234734e360d83b74f120cbb b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/834911912234734e360d83b74f120cbb new file mode 100644 index 0000000000000000000000000000000000000000..839c89eeb35290a3cc8bd8b45d0455de9f4bf689 Binary files /dev/null and b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/834911912234734e360d83b74f120cbb differ diff --git a/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/9db3aae2c824f358c75bbbdf13fac62d b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/9db3aae2c824f358c75bbbdf13fac62d new file mode 100644 index 0000000000000000000000000000000000000000..3b51373ca9a97e152acc57053a04d3822ab2b6db --- /dev/null +++ b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/9db3aae2c824f358c75bbbdf13fac62d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e809f89a53c467b649abb6c44e24a0b811501e9d72d95f6611d45d17954285 +size 3883780 diff --git a/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/a2d2f3b16221b992f78d68bb2a69657e b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/a2d2f3b16221b992f78d68bb2a69657e new file mode 100644 index 0000000000000000000000000000000000000000..c40c4950104d5a206a3f7e1137352a3cefab660a Binary files /dev/null and b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/d/a2d2f3b16221b992f78d68bb2a69657e differ diff --git a/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..004fda2a03d1f3dc10ae5dbfcfbf90c38b2c2b05 Binary files /dev/null and b/owt/uniform/checkpoints_ema/250000/state/ocdbt.process_0/manifest.ocdbt differ diff --git a/text8/binary_absorbing/.hydra/config.yaml b/text8/binary_absorbing/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb63066828ec9a1fae8a714820a57068c3867908 --- /dev/null +++ b/text8/binary_absorbing/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: binary_absorbing + objective: wxent + schedule: log + t_min: 0.001 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 768 + num_layers: 12 + num_heads: 12 + mlp_dim: 3072 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 1000000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: false + dataset_path: data/text8 + tokenizer_name: data/text8/tokenizer + max_seq_len: 256 diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/_CHECKPOINT_METADATA b/text8/binary_absorbing/checkpoints_ema/1000000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..ae18c2079b4c54438506bc27ec5764205ddf23df --- /dev/null +++ b/text8/binary_absorbing/checkpoints_ema/1000000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789302351032403521, "commit_timestamp_nsecs": 1789302371478168486, "custom_metadata": {}} \ No newline at end of file diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/state/_METADATA b/text8/binary_absorbing/checkpoints_ema/1000000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..a4bc4dbb865b9d31b2f9a20001693d360ce4c4a0 --- /dev/null +++ b/text8/binary_absorbing/checkpoints_ema/1000000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [256, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [29, 768]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1536]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 1536]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [32]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 32]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/state/_sharding b/text8/binary_absorbing/checkpoints_ema/1000000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..2ced0ba60ec2ffa96c352791431a42482d422ef7 --- /dev/null +++ b/text8/binary_absorbing/checkpoints_ema/1000000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}"} \ No newline at end of file diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/state/array_metadatas/process_0 b/text8/binary_absorbing/checkpoints_ema/1000000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..74a001b06255f88fdad38f94b060f905a8d4770f --- /dev/null +++ b/text8/binary_absorbing/checkpoints_ema/1000000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [256, 128], "chunk_shape": [256, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [29, 768], "chunk_shape": [29, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1536], "chunk_shape": [1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [128, 1536], "chunk_shape": [128, 1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [32], "chunk_shape": [32], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [768, 32], "chunk_shape": [768, 32], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}]} \ No newline at end of file diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/state/d/f406c64bd49d13baf0885d0e70dfe603 b/text8/binary_absorbing/checkpoints_ema/1000000/state/d/f406c64bd49d13baf0885d0e70dfe603 new file mode 100644 index 0000000000000000000000000000000000000000..1dfca470f0a22293ab67413ee0d56b6677ecc9ac Binary files /dev/null and b/text8/binary_absorbing/checkpoints_ema/1000000/state/d/f406c64bd49d13baf0885d0e70dfe603 differ diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/state/manifest.ocdbt b/text8/binary_absorbing/checkpoints_ema/1000000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..571d03c072346767abcd54f12d8619fc4fa7e624 Binary files /dev/null and b/text8/binary_absorbing/checkpoints_ema/1000000/state/manifest.ocdbt differ diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/03fa2fdee7a8b473b64ef0bc7f990c35 b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/03fa2fdee7a8b473b64ef0bc7f990c35 new file mode 100644 index 0000000000000000000000000000000000000000..344e47e1ddb351c4a8a91d021b624feed7aeea7d Binary files /dev/null and b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/03fa2fdee7a8b473b64ef0bc7f990c35 differ diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/b6d45a6e08a3adfce23fe173b50b483a b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/b6d45a6e08a3adfce23fe173b50b483a new file mode 100644 index 0000000000000000000000000000000000000000..04b8019296a41485d435882936174111e815529a --- /dev/null +++ b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/b6d45a6e08a3adfce23fe173b50b483a @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3c3857284ca6ab695b307fb9dab329df25ac7c32a3d93b4dc1a7d4da1857232 +size 72327168 diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/bd7b0fe12cc853e857127a9c6c3f8b07 b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/bd7b0fe12cc853e857127a9c6c3f8b07 new file mode 100644 index 0000000000000000000000000000000000000000..ea2a5d1f85096f950662bd20fa7a9f291df08c39 Binary files /dev/null and b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/bd7b0fe12cc853e857127a9c6c3f8b07 differ diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/c14ea4fd91c09ebb390aa17edb2e647b b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/c14ea4fd91c09ebb390aa17edb2e647b new file mode 100644 index 0000000000000000000000000000000000000000..8026470fd0ae425e2eecfe47fa0e15793c7ddb53 --- /dev/null +++ b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/c14ea4fd91c09ebb390aa17edb2e647b @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccc3c92bf81913114267a9184222499f4888e0b81e0c6f7914ffce3e798e3a8e +size 268722176 diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/fba552e9dfbcec2f5571cafbf0a95d35 b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/fba552e9dfbcec2f5571cafbf0a95d35 new file mode 100644 index 0000000000000000000000000000000000000000..ebdbcc7d34977b25b81ee47338c670ec4d792e5d --- /dev/null +++ b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/d/fba552e9dfbcec2f5571cafbf0a95d35 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa5d7e28a98a10b01334b77bcbec33f92b4266c1d43ad6c2cbef924cdcd3b5f7 +size 2219021 diff --git a/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..841a0aaea215951c134f1732a544bfef80bdadb0 Binary files /dev/null and b/text8/binary_absorbing/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt differ diff --git a/text8/binary_symmetric/.hydra/config.yaml b/text8/binary_symmetric/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b5e34c27bad5b4bab8823a283ae867095c78aa7 --- /dev/null +++ b/text8/binary_symmetric/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: binary_symmetric + objective: elbo + schedule: log + t_min: 0.001 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 768 + num_layers: 12 + num_heads: 12 + mlp_dim: 3072 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 1000000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: false + dataset_path: data/text8 + tokenizer_name: data/text8/tokenizer + max_seq_len: 256 diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/_CHECKPOINT_METADATA b/text8/binary_symmetric/checkpoints_ema/1000000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..0319bd0a9aca702011d93d0533bd21bb98a2c135 --- /dev/null +++ b/text8/binary_symmetric/checkpoints_ema/1000000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789236457225905223, "commit_timestamp_nsecs": 1789236459770744129, "custom_metadata": {}} \ No newline at end of file diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/state/_METADATA b/text8/binary_symmetric/checkpoints_ema/1000000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..a4bc4dbb865b9d31b2f9a20001693d360ce4c4a0 --- /dev/null +++ b/text8/binary_symmetric/checkpoints_ema/1000000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [256, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [29, 768]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1536]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 1536]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [32]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 32]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/state/_sharding b/text8/binary_symmetric/checkpoints_ema/1000000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..2ced0ba60ec2ffa96c352791431a42482d422ef7 --- /dev/null +++ b/text8/binary_symmetric/checkpoints_ema/1000000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}"} \ No newline at end of file diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/state/array_metadatas/process_0 b/text8/binary_symmetric/checkpoints_ema/1000000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..74a001b06255f88fdad38f94b060f905a8d4770f --- /dev/null +++ b/text8/binary_symmetric/checkpoints_ema/1000000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [256, 128], "chunk_shape": [256, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [29, 768], "chunk_shape": [29, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1536], "chunk_shape": [1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [128, 1536], "chunk_shape": [128, 1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [32], "chunk_shape": [32], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [768, 32], "chunk_shape": [768, 32], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}]} \ No newline at end of file diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/state/d/eed0c6f68af6bceb3da8dd239970288f b/text8/binary_symmetric/checkpoints_ema/1000000/state/d/eed0c6f68af6bceb3da8dd239970288f new file mode 100644 index 0000000000000000000000000000000000000000..5ddefccd870501dd1a24552129670e114b28754f Binary files /dev/null and b/text8/binary_symmetric/checkpoints_ema/1000000/state/d/eed0c6f68af6bceb3da8dd239970288f differ diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/state/manifest.ocdbt b/text8/binary_symmetric/checkpoints_ema/1000000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..a72aa63c452d34845fcdba8566709e5817138fb1 Binary files /dev/null and b/text8/binary_symmetric/checkpoints_ema/1000000/state/manifest.ocdbt differ diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/038873e5c4f3aff476bce1260152c3b3 b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/038873e5c4f3aff476bce1260152c3b3 new file mode 100644 index 0000000000000000000000000000000000000000..b6d6e7810b075bd9e01f85a295ac17605a9cc1c6 --- /dev/null +++ b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/038873e5c4f3aff476bce1260152c3b3 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a08bbf36de528b069bfeec94a5309d230c1310075c4ddeef5e26ef3da37f39ea +size 343240704 diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/407bc2d34bbe2dac6845330f23d149a1 b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/407bc2d34bbe2dac6845330f23d149a1 new file mode 100644 index 0000000000000000000000000000000000000000..a35be0dc5da3fc21f7235b6e91259a05168e3b01 Binary files /dev/null and b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/407bc2d34bbe2dac6845330f23d149a1 differ diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/601d462ee22630519067097837988d2f b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/601d462ee22630519067097837988d2f new file mode 100644 index 0000000000000000000000000000000000000000..8bb163cd68d58fed84daca5f0a874430144a36b0 Binary files /dev/null and b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/601d462ee22630519067097837988d2f differ diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/911ddd9d7bbfb220b9a25a0de1a9c7c6 b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/911ddd9d7bbfb220b9a25a0de1a9c7c6 new file mode 100644 index 0000000000000000000000000000000000000000..ab030180bf8e7041cedd0e81a20a191e21481f13 Binary files /dev/null and b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/911ddd9d7bbfb220b9a25a0de1a9c7c6 differ diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/ee048d3b117d219211e450d2ffb290c4 b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/ee048d3b117d219211e450d2ffb290c4 new file mode 100644 index 0000000000000000000000000000000000000000..cda20697230e1c2cab5d0cb0f50facc6a4100774 Binary files /dev/null and b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/d/ee048d3b117d219211e450d2ffb290c4 differ diff --git a/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..6444e40c0d6cbc02b3560790d6bbcc49bc9d0f4d Binary files /dev/null and b/text8/binary_symmetric/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt differ diff --git a/text8/masked/.hydra/config.yaml b/text8/masked/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bedba02d400c1ade0d8c1e98856925ed40d693b1 --- /dev/null +++ b/text8/masked/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: masked + objective: xent + schedule: linear + t_min: 0.001 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 768 + num_layers: 12 + num_heads: 12 + mlp_dim: 3072 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 1000000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: false + dataset_path: data/text8 + tokenizer_name: data/text8/tokenizer + max_seq_len: 256 diff --git a/text8/masked/checkpoints_ema/1000000/_CHECKPOINT_METADATA b/text8/masked/checkpoints_ema/1000000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..cb455ac3f44e6d46af421c43540d624aa95df015 --- /dev/null +++ b/text8/masked/checkpoints_ema/1000000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789180244204161621, "commit_timestamp_nsecs": 1789180260154449184, "custom_metadata": {}} \ No newline at end of file diff --git a/text8/masked/checkpoints_ema/1000000/state/_METADATA b/text8/masked/checkpoints_ema/1000000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..a4bc4dbb865b9d31b2f9a20001693d360ce4c4a0 --- /dev/null +++ b/text8/masked/checkpoints_ema/1000000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [256, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [29, 768]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1536]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 1536]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [32]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 32]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/text8/masked/checkpoints_ema/1000000/state/_sharding b/text8/masked/checkpoints_ema/1000000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..2ced0ba60ec2ffa96c352791431a42482d422ef7 --- /dev/null +++ b/text8/masked/checkpoints_ema/1000000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}"} \ No newline at end of file diff --git a/text8/masked/checkpoints_ema/1000000/state/array_metadatas/process_0 b/text8/masked/checkpoints_ema/1000000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..74a001b06255f88fdad38f94b060f905a8d4770f --- /dev/null +++ b/text8/masked/checkpoints_ema/1000000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [256, 128], "chunk_shape": [256, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [29, 768], "chunk_shape": [29, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1536], "chunk_shape": [1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [128, 1536], "chunk_shape": [128, 1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [32], "chunk_shape": [32], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [768, 32], "chunk_shape": [768, 32], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}]} \ No newline at end of file diff --git a/text8/masked/checkpoints_ema/1000000/state/d/cb59cbeefdd12ae309074f944d59c278 b/text8/masked/checkpoints_ema/1000000/state/d/cb59cbeefdd12ae309074f944d59c278 new file mode 100644 index 0000000000000000000000000000000000000000..3cc69b8c5e97ddf56201ea7eeaff4061f723b475 Binary files /dev/null and b/text8/masked/checkpoints_ema/1000000/state/d/cb59cbeefdd12ae309074f944d59c278 differ diff --git a/text8/masked/checkpoints_ema/1000000/state/manifest.ocdbt b/text8/masked/checkpoints_ema/1000000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..0bbf14425e4779a96952268cef2a3156cd5554ff Binary files /dev/null and b/text8/masked/checkpoints_ema/1000000/state/manifest.ocdbt differ diff --git a/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/0511ed773775dd2efbe8f7ad95ecfdd5 b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/0511ed773775dd2efbe8f7ad95ecfdd5 new file mode 100644 index 0000000000000000000000000000000000000000..d35bf8de691e43ca632460ac50eee0be9f0c1932 --- /dev/null +++ b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/0511ed773775dd2efbe8f7ad95ecfdd5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa37484aeb8814a9f13a8c1e6c719dabd2f43efecba533044fe946171a514e42 +size 334123008 diff --git a/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/31fbd64151e1c27957b25b54f68eeddf b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/31fbd64151e1c27957b25b54f68eeddf new file mode 100644 index 0000000000000000000000000000000000000000..23f708947af9e3d61a1784c0df8fb3b12055aba9 --- /dev/null +++ b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/31fbd64151e1c27957b25b54f68eeddf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:990498f61b1f29e91682f382b4c5bb27419391baf80fd32c174dabd633551b02 +size 8978432 diff --git a/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/422b14d17bc7167d95887adc3152a4b9 b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/422b14d17bc7167d95887adc3152a4b9 new file mode 100644 index 0000000000000000000000000000000000000000..f3e532cbfa4193eef461237e9d3fac3aed65aeef Binary files /dev/null and b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/422b14d17bc7167d95887adc3152a4b9 differ diff --git a/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/55beac1f5ad2f29434b0c75c9440161e b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/55beac1f5ad2f29434b0c75c9440161e new file mode 100644 index 0000000000000000000000000000000000000000..344e47e1ddb351c4a8a91d021b624feed7aeea7d Binary files /dev/null and b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/55beac1f5ad2f29434b0c75c9440161e differ diff --git a/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/8ea2181e8ee2011b81fff647dcaceb5d b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/8ea2181e8ee2011b81fff647dcaceb5d new file mode 100644 index 0000000000000000000000000000000000000000..bb3b78c97dae69433cbf66f2e6b1d15d71d152fb Binary files /dev/null and b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/d/8ea2181e8ee2011b81fff647dcaceb5d differ diff --git a/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..d2a7e31ed6170e2837033f2f105fab7e77277e99 Binary files /dev/null and b/text8/masked/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt differ diff --git a/text8/uniform/.hydra/config.yaml b/text8/uniform/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08dae7938b985f2ce92899bfd113420d1b79cd47 --- /dev/null +++ b/text8/uniform/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: uniform + objective: elbo + schedule: linear + t_min: 0.001 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 768 + num_layers: 12 + num_heads: 12 + mlp_dim: 3072 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 1000000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: false + dataset_path: data/text8 + tokenizer_name: data/text8/tokenizer + max_seq_len: 256 diff --git a/text8/uniform/checkpoints_ema/1000000/_CHECKPOINT_METADATA b/text8/uniform/checkpoints_ema/1000000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..d68b72ed878a0c59be5306f746d3377e1b51ebe4 --- /dev/null +++ b/text8/uniform/checkpoints_ema/1000000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789272744209449910, "commit_timestamp_nsecs": 1789272758559897604, "custom_metadata": {}} \ No newline at end of file diff --git a/text8/uniform/checkpoints_ema/1000000/state/_METADATA b/text8/uniform/checkpoints_ema/1000000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..a4bc4dbb865b9d31b2f9a20001693d360ce4c4a0 --- /dev/null +++ b/text8/uniform/checkpoints_ema/1000000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [256, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [29, 768]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1536]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 1536]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [32]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 32]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/text8/uniform/checkpoints_ema/1000000/state/_sharding b/text8/uniform/checkpoints_ema/1000000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..2ced0ba60ec2ffa96c352791431a42482d422ef7 --- /dev/null +++ b/text8/uniform/checkpoints_ema/1000000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}"} \ No newline at end of file diff --git a/text8/uniform/checkpoints_ema/1000000/state/array_metadatas/process_0 b/text8/uniform/checkpoints_ema/1000000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..74a001b06255f88fdad38f94b060f905a8d4770f --- /dev/null +++ b/text8/uniform/checkpoints_ema/1000000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [256, 128], "chunk_shape": [256, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [29, 768], "chunk_shape": [29, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1536], "chunk_shape": [1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [128, 1536], "chunk_shape": [128, 1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [32], "chunk_shape": [32], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [768, 32], "chunk_shape": [768, 32], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}]} \ No newline at end of file diff --git a/text8/uniform/checkpoints_ema/1000000/state/d/ee507132cf948c4d8c9659267c8f5767 b/text8/uniform/checkpoints_ema/1000000/state/d/ee507132cf948c4d8c9659267c8f5767 new file mode 100644 index 0000000000000000000000000000000000000000..6ad1006188f2ee9b6e757291df69c76e697de7a6 Binary files /dev/null and b/text8/uniform/checkpoints_ema/1000000/state/d/ee507132cf948c4d8c9659267c8f5767 differ diff --git a/text8/uniform/checkpoints_ema/1000000/state/manifest.ocdbt b/text8/uniform/checkpoints_ema/1000000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..43fb4ddcc6cfe3a58cd26e3377d485e616db28ea Binary files /dev/null and b/text8/uniform/checkpoints_ema/1000000/state/manifest.ocdbt differ diff --git a/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/764aa799191684deed3790948a34d993 b/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/764aa799191684deed3790948a34d993 new file mode 100644 index 0000000000000000000000000000000000000000..07c3991a57ed8d62961d18e7e4ceff388a180a8d --- /dev/null +++ b/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/764aa799191684deed3790948a34d993 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b913bc2bd63811d24ae2ec14f1aa08f6fc1fab59bb186d86b3a624f702473c53 +size 336302080 diff --git a/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/887531dd96fe34101ec9571599ba7832 b/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/887531dd96fe34101ec9571599ba7832 new file mode 100644 index 0000000000000000000000000000000000000000..b3211b2ae88c423b1f7a8b7edf5bbb4f8021cf15 Binary files /dev/null and b/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/887531dd96fe34101ec9571599ba7832 differ diff --git a/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/9992e06dbbc03954ae4aa2764c028315 b/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/9992e06dbbc03954ae4aa2764c028315 new file mode 100644 index 0000000000000000000000000000000000000000..3375dee3d4e4dfa337432f758a78b5e887c65299 --- /dev/null +++ b/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/9992e06dbbc03954ae4aa2764c028315 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4636fbbb22f633a03de0d8ee020bb3bcc14b756038e2ee50880176b744dcb2f6 +size 6635520 diff --git a/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/a031af01a50b11b24d1676a7cc493b55 b/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/a031af01a50b11b24d1676a7cc493b55 new file mode 100644 index 0000000000000000000000000000000000000000..344e47e1ddb351c4a8a91d021b624feed7aeea7d Binary files /dev/null and b/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/d/a031af01a50b11b24d1676a7cc493b55 differ diff --git a/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt b/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..badf41e08cbbaa51987b040967a6be9a43186b52 Binary files /dev/null and b/text8/uniform/checkpoints_ema/1000000/state/ocdbt.process_0/manifest.ocdbt differ diff --git a/tinystories/binary_absorbing/.hydra/config.yaml b/tinystories/binary_absorbing/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e669e10946d819d917283131fbbec63a5461eac2 --- /dev/null +++ b/tinystories/binary_absorbing/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: binary_absorbing + objective: elbo + schedule: log + t_min: 0.0005 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 768 + num_layers: 12 + num_heads: 12 + mlp_dim: 3072 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 500000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: true + dataset_path: data/tinystories + tokenizer_name: rodcasnog/tinystories-bpe-1k + max_seq_len: 512 diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/_CHECKPOINT_METADATA b/tinystories/binary_absorbing/checkpoints_ema/500000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..3cbcfa4539fb38426478af0ead1519a10e6e1acf --- /dev/null +++ b/tinystories/binary_absorbing/checkpoints_ema/500000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789556224718807887, "commit_timestamp_nsecs": 1789556247970664114, "custom_metadata": {}} \ No newline at end of file diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/state/_METADATA b/tinystories/binary_absorbing/checkpoints_ema/500000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..2d792ab3ca44b7ec07db13867ab6604f2d76be4e --- /dev/null +++ b/tinystories/binary_absorbing/checkpoints_ema/500000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [256, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1026, 768]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1536]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 1536]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1024]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 1024]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/state/_sharding b/tinystories/binary_absorbing/checkpoints_ema/500000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..2ced0ba60ec2ffa96c352791431a42482d422ef7 --- /dev/null +++ b/tinystories/binary_absorbing/checkpoints_ema/500000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}"} \ No newline at end of file diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/state/array_metadatas/process_0 b/tinystories/binary_absorbing/checkpoints_ema/500000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..b4d02a076b7c3291b176ccf495f0ca8f8bcc852b --- /dev/null +++ b/tinystories/binary_absorbing/checkpoints_ema/500000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [256, 128], "chunk_shape": [256, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [1026, 768], "chunk_shape": [1026, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1536], "chunk_shape": [1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [128, 1536], "chunk_shape": [128, 1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [1024], "chunk_shape": [1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [768, 1024], "chunk_shape": [768, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}]} \ No newline at end of file diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/state/d/f0f0deb879486d70fd5fe9156c809894 b/tinystories/binary_absorbing/checkpoints_ema/500000/state/d/f0f0deb879486d70fd5fe9156c809894 new file mode 100644 index 0000000000000000000000000000000000000000..40f2dbc387095d427083e03ef6e8225855d46003 Binary files /dev/null and b/tinystories/binary_absorbing/checkpoints_ema/500000/state/d/f0f0deb879486d70fd5fe9156c809894 differ diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/state/manifest.ocdbt b/tinystories/binary_absorbing/checkpoints_ema/500000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..27e1fd6a65a588f93fe1ecba38b9d5b72ce0a2b7 Binary files /dev/null and b/tinystories/binary_absorbing/checkpoints_ema/500000/state/manifest.ocdbt differ diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/065ec2be3d26fef4dbbc9a9050f17a71 b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/065ec2be3d26fef4dbbc9a9050f17a71 new file mode 100644 index 0000000000000000000000000000000000000000..c13abcb93c32177f32619f11849f11a55bdc8606 --- /dev/null +++ b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/065ec2be3d26fef4dbbc9a9050f17a71 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:219114363dbd492d2a78dbf6d59345a3a49b67ee6760153842f111320451e772 +size 72232960 diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/148a233581cc705768f81424d855fc35 b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/148a233581cc705768f81424d855fc35 new file mode 100644 index 0000000000000000000000000000000000000000..f3e532cbfa4193eef461237e9d3fac3aed65aeef Binary files /dev/null and b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/148a233581cc705768f81424d855fc35 differ diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/3274ff5e02e402c544b9a447121b27f4 b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/3274ff5e02e402c544b9a447121b27f4 new file mode 100644 index 0000000000000000000000000000000000000000..535d281d39fc036ed84d5f87c8f8aed38b0904f4 --- /dev/null +++ b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/3274ff5e02e402c544b9a447121b27f4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c92305e606b09054b458d63197644be055695c26c90c345e4afeb8385f665db8 +size 267694080 diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/71120e92864115c1064133a6a47785e8 b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/71120e92864115c1064133a6a47785e8 new file mode 100644 index 0000000000000000000000000000000000000000..79c6812fa69506d2ed7349af6080ac8b9c1f7718 Binary files /dev/null and b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/71120e92864115c1064133a6a47785e8 differ diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/9f27210eedd8af7596fe78838a126b12 b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/9f27210eedd8af7596fe78838a126b12 new file mode 100644 index 0000000000000000000000000000000000000000..e7fa354e3344259f22c32951b9bd40dd2d503942 --- /dev/null +++ b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/d/9f27210eedd8af7596fe78838a126b12 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:446a234c819e7eb9174aaf7d3b7e116fea364311c9bc76f0c64ec4d5a2ae5100 +size 8740864 diff --git a/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..2d5e698963513bdebe0c3c8f77263f42322600d0 Binary files /dev/null and b/tinystories/binary_absorbing/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt differ diff --git a/tinystories/binary_symmetric/.hydra/config.yaml b/tinystories/binary_symmetric/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..617752c645b74766590cb63797973970c8923346 --- /dev/null +++ b/tinystories/binary_symmetric/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: binary_symmetric + objective: elbo + schedule: log + t_min: 0.0005 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 768 + num_layers: 12 + num_heads: 12 + mlp_dim: 3072 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 500000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: true + dataset_path: data/tinystories + tokenizer_name: rodcasnog/tinystories-bpe-1k + max_seq_len: 512 diff --git a/tinystories/binary_symmetric/checkpoints_ema/500000/_CHECKPOINT_METADATA b/tinystories/binary_symmetric/checkpoints_ema/500000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..81f1826ba9b4a771aacbf943d1130ec41dcaab2f --- /dev/null +++ b/tinystories/binary_symmetric/checkpoints_ema/500000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789557292977202177, "commit_timestamp_nsecs": 1789557309542192434, "custom_metadata": {}} \ No newline at end of file diff --git a/tinystories/binary_symmetric/checkpoints_ema/500000/state/_METADATA b/tinystories/binary_symmetric/checkpoints_ema/500000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..2d792ab3ca44b7ec07db13867ab6604f2d76be4e --- /dev/null +++ b/tinystories/binary_symmetric/checkpoints_ema/500000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [256, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1026, 768]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1536]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 1536]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1024]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 1024]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/tinystories/binary_symmetric/checkpoints_ema/500000/state/_sharding b/tinystories/binary_symmetric/checkpoints_ema/500000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..2ced0ba60ec2ffa96c352791431a42482d422ef7 --- /dev/null +++ b/tinystories/binary_symmetric/checkpoints_ema/500000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}"} \ No newline at end of file diff --git a/tinystories/binary_symmetric/checkpoints_ema/500000/state/array_metadatas/process_0 b/tinystories/binary_symmetric/checkpoints_ema/500000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..b4d02a076b7c3291b176ccf495f0ca8f8bcc852b --- /dev/null +++ b/tinystories/binary_symmetric/checkpoints_ema/500000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [256, 128], "chunk_shape": [256, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [1026, 768], "chunk_shape": [1026, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1536], "chunk_shape": [1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [128, 1536], "chunk_shape": [128, 1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [1024], "chunk_shape": [1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [768, 1024], "chunk_shape": [768, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}]} \ No newline at end of file diff --git a/tinystories/binary_symmetric/checkpoints_ema/500000/state/d/62a690ce9e21661a7f09d734b2d38940 b/tinystories/binary_symmetric/checkpoints_ema/500000/state/d/62a690ce9e21661a7f09d734b2d38940 new file mode 100644 index 0000000000000000000000000000000000000000..eb31bbd41f0f4f4780863c45645f7126dec0e5d0 Binary files /dev/null and b/tinystories/binary_symmetric/checkpoints_ema/500000/state/d/62a690ce9e21661a7f09d734b2d38940 differ diff --git a/tinystories/binary_symmetric/checkpoints_ema/500000/state/manifest.ocdbt b/tinystories/binary_symmetric/checkpoints_ema/500000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..982f9d4cd59b01a3594e6e2995abac96d35093ed Binary files /dev/null and b/tinystories/binary_symmetric/checkpoints_ema/500000/state/manifest.ocdbt differ diff --git a/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/15f1356815fabf8024fca8cee55adc78 b/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/15f1356815fabf8024fca8cee55adc78 new file mode 100644 index 0000000000000000000000000000000000000000..e6cfb889ce20ccd14183b3b7470acd9876559043 --- /dev/null +++ b/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/15f1356815fabf8024fca8cee55adc78 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee72ed67d7b40e2f66343277c28f501270fadef4aba209f6eeacc7340c311b51 +size 282791936 diff --git a/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/22ced4c87d659b34ab0fbe61a47027d5 b/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/22ced4c87d659b34ab0fbe61a47027d5 new file mode 100644 index 0000000000000000000000000000000000000000..039cf648bc41f7f1815b6e745c14bbd9c07158d3 Binary files /dev/null and b/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/22ced4c87d659b34ab0fbe61a47027d5 differ diff --git a/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/68757985fb3b0f9dbc7d2a51d124fd5b b/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/68757985fb3b0f9dbc7d2a51d124fd5b new file mode 100644 index 0000000000000000000000000000000000000000..6970597a54a4165a8e7eb07886574fad59262926 --- /dev/null +++ b/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/68757985fb3b0f9dbc7d2a51d124fd5b @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be8394802b8826cc833e6d2d2b2c4e99fcedc5d016d887003a6c43ce7bda6f68 +size 65892352 diff --git a/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/e3863cc89e17845f21d25618b8c49801 b/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/e3863cc89e17845f21d25618b8c49801 new file mode 100644 index 0000000000000000000000000000000000000000..79c6812fa69506d2ed7349af6080ac8b9c1f7718 Binary files /dev/null and b/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/d/e3863cc89e17845f21d25618b8c49801 differ diff --git a/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt b/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..2c11d8bcdf67719aa82c14e9fcf9c16852e17375 Binary files /dev/null and b/tinystories/binary_symmetric/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt differ diff --git a/tinystories/masked/.hydra/config.yaml b/tinystories/masked/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0460b8e40bfec2b5e999f50c60533287f2f875be --- /dev/null +++ b/tinystories/masked/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: masked + objective: elbo + schedule: linear + t_min: 0.0005 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 768 + num_layers: 12 + num_heads: 12 + mlp_dim: 3072 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 500000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: true + dataset_path: data/tinystories + tokenizer_name: rodcasnog/tinystories-bpe-1k + max_seq_len: 512 diff --git a/tinystories/masked/checkpoints_ema/500000/_CHECKPOINT_METADATA b/tinystories/masked/checkpoints_ema/500000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..c498003a70a4319a1e43ebdc76eeb29536b9f0bd --- /dev/null +++ b/tinystories/masked/checkpoints_ema/500000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789509097890624127, "commit_timestamp_nsecs": 1789509124038927159, "custom_metadata": {}} \ No newline at end of file diff --git a/tinystories/masked/checkpoints_ema/500000/state/_METADATA b/tinystories/masked/checkpoints_ema/500000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..2d792ab3ca44b7ec07db13867ab6604f2d76be4e --- /dev/null +++ b/tinystories/masked/checkpoints_ema/500000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [256, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1026, 768]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1536]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 1536]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1024]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 1024]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/tinystories/masked/checkpoints_ema/500000/state/_sharding b/tinystories/masked/checkpoints_ema/500000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..2ced0ba60ec2ffa96c352791431a42482d422ef7 --- /dev/null +++ b/tinystories/masked/checkpoints_ema/500000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}"} \ No newline at end of file diff --git a/tinystories/masked/checkpoints_ema/500000/state/array_metadatas/process_0 b/tinystories/masked/checkpoints_ema/500000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..b4d02a076b7c3291b176ccf495f0ca8f8bcc852b --- /dev/null +++ b/tinystories/masked/checkpoints_ema/500000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [256, 128], "chunk_shape": [256, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [1026, 768], "chunk_shape": [1026, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1536], "chunk_shape": [1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [128, 1536], "chunk_shape": [128, 1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [1024], "chunk_shape": [1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [768, 1024], "chunk_shape": [768, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}]} \ No newline at end of file diff --git a/tinystories/masked/checkpoints_ema/500000/state/d/e3f1098c0e7f4bfcc10901684fa25f36 b/tinystories/masked/checkpoints_ema/500000/state/d/e3f1098c0e7f4bfcc10901684fa25f36 new file mode 100644 index 0000000000000000000000000000000000000000..e6e9e7a0d31c6f6c9454199dc2b3f6980fefe07b Binary files /dev/null and b/tinystories/masked/checkpoints_ema/500000/state/d/e3f1098c0e7f4bfcc10901684fa25f36 differ diff --git a/tinystories/masked/checkpoints_ema/500000/state/manifest.ocdbt b/tinystories/masked/checkpoints_ema/500000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..7431f9bde410a0d27ebd870efbf8cdfcabe2e0d6 Binary files /dev/null and b/tinystories/masked/checkpoints_ema/500000/state/manifest.ocdbt differ diff --git a/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/62864d434e45c5f6cf8e3945328b617d b/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/62864d434e45c5f6cf8e3945328b617d new file mode 100644 index 0000000000000000000000000000000000000000..aa19dde01ce322884805871a873432eca5047fa1 Binary files /dev/null and b/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/62864d434e45c5f6cf8e3945328b617d differ diff --git a/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/a4fbec2e52c1c7bf2d2e05d2a90ebae6 b/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/a4fbec2e52c1c7bf2d2e05d2a90ebae6 new file mode 100644 index 0000000000000000000000000000000000000000..57a6b2a80bc6e861ca1bf7f31bad2392dd03f7de --- /dev/null +++ b/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/a4fbec2e52c1c7bf2d2e05d2a90ebae6 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb729fe2064403e19ab449662e5dc21dc75043f69cb54a396ff6febd969f48e6 +size 346152960 diff --git a/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/c9d5308de0df3c863e62db6465227bed b/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/c9d5308de0df3c863e62db6465227bed new file mode 100644 index 0000000000000000000000000000000000000000..d47a7a2a7b605fadc18c3f396653a8f786fe7c21 --- /dev/null +++ b/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/c9d5308de0df3c863e62db6465227bed @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72b8dce181844c1772bd44611b387b4d04831b444b8d0f127934602eb2732126 +size 2185029 diff --git a/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/de8239f88bfda40744f9b772c321c375 b/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/de8239f88bfda40744f9b772c321c375 new file mode 100644 index 0000000000000000000000000000000000000000..79c6812fa69506d2ed7349af6080ac8b9c1f7718 Binary files /dev/null and b/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/d/de8239f88bfda40744f9b772c321c375 differ diff --git a/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt b/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..2a970d009dec0bbb4bb62c4ffac24e164a73a25f Binary files /dev/null and b/tinystories/masked/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt differ diff --git a/tinystories/uniform/.hydra/config.yaml b/tinystories/uniform/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9bfed3d348f3037f4b43682cb8f58a153f2be51 --- /dev/null +++ b/tinystories/uniform/.hydra/config.yaml @@ -0,0 +1,43 @@ +diffusion: + kernel: uniform + objective: elbo + schedule: linear + t_min: 0.0005 + t_max: 0.9999 +model: + architecture: dit + hidden_dim: 768 + num_layers: 12 + num_heads: 12 + mlp_dim: 3072 + cond_dim: 128 + dtype: bfloat16 + clue_embedding: false +training: + seed: 0 + num_steps: 500000 + global_batch_size: 512 + learning_rate: 0.0003 + warmup_steps: 2500 + weight_decay: 0.02 + adam_b1: 0.9 + adam_b2: 0.999 + max_grad_norm: 1.0 + ema_decay: 0.9999 + log_steps: 100 + val_steps: 1000 + val_batches: 32 + save_every: 2500 +checkpoint: + max_to_keep: 5 + ema_max_to_keep: 5 + keep_period: 100000 +wandb: + project: binary-discrete-diffusion + mode: disabled +data: + source: text + padded: true + dataset_path: data/tinystories + tokenizer_name: rodcasnog/tinystories-bpe-1k + max_seq_len: 512 diff --git a/tinystories/uniform/checkpoints_ema/500000/_CHECKPOINT_METADATA b/tinystories/uniform/checkpoints_ema/500000/_CHECKPOINT_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..2243966b92fdeedafb26785e9fed794de4995f50 --- /dev/null +++ b/tinystories/uniform/checkpoints_ema/500000/_CHECKPOINT_METADATA @@ -0,0 +1 @@ +{"item_handlers": {"state": "orbax.checkpoint._src.handlers.pytree_checkpoint_handler.PyTreeCheckpointHandler"}, "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789551843270353101, "commit_timestamp_nsecs": 1789551844855412720, "custom_metadata": {}} \ No newline at end of file diff --git a/tinystories/uniform/checkpoints_ema/500000/state/_METADATA b/tinystories/uniform/checkpoints_ema/500000/state/_METADATA new file mode 100644 index 0000000000000000000000000000000000000000..2d792ab3ca44b7ec07db13867ab6604f2d76be4e --- /dev/null +++ b/tinystories/uniform/checkpoints_ema/500000/state/_METADATA @@ -0,0 +1 @@ +{"tree_metadata": {"('params', 'blocks', '0', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '0', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '0', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '0', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '0', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '0', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '0', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '0', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '0', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "0", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '1', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '1', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '1', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '1', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '1', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '1', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '1', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '1', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "1", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '2', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '2', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '2', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '2', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '2', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '2', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '2', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '2', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "2", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '3', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '3', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '3', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '3', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '3', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '3', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '3', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '3', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "3", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '4', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '4', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '4', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '4', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '4', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '4', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '4', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '4', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "4", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '5', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '5', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '5', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '5', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '5', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '5', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '5', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '5', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "5", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '6', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '6', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '6', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '6', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '6', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '6', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '6', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '6', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "6", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '7', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '7', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '7', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '7', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '7', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '7', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '7', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '7', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "7", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '8', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '8', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '8', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '8', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '8', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '8', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '8', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '8', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "8", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '9', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '9', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '9', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '9', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '9', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '9', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '9', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '9', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "9", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '10', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '10', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '10', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '10', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '10', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '10', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '10', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '10', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "10", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [4608]}}, "('params', 'blocks', '11', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 4608]}}, "('params', 'blocks', '11', 'attn_out', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_out", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [12, 64, 768]}}, "('params', 'blocks', '11', 'attn_qkv', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "attn_qkv", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 12, 3, 64]}}, "('params', 'blocks', '11', 'mlp_fc1', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072]}}, "('params', 'blocks', '11', 'mlp_fc1', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc1", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 3072]}}, "('params', 'blocks', '11', 'mlp_fc2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'mlp_fc2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "mlp_fc2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [3072, 768]}}, "('params', 'blocks', '11', 'norm1', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm1", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'blocks', '11', 'norm2', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "blocks", "key_type": 2}, {"key": "11", "key_type": 2}, {"key": "norm2", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_0', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_0", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [256, 128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128]}}, "('params', 'conditioner', 'sigma_map', 'mlp_2', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "sigma_map", "key_type": 2}, {"key": "mlp_2", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 128]}}, "('params', 'conditioner', 'vocab_embed', 'embedding', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "conditioner", "key_type": 2}, {"key": "vocab_embed", "key_type": 2}, {"key": "embedding", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1026, 768]}}, "('params', 'output_layer', 'adaLN_modulation', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1536]}}, "('params', 'output_layer', 'adaLN_modulation', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "adaLN_modulation", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [128, 1536]}}, "('params', 'output_layer', 'linear', 'bias', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "bias", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [1024]}}, "('params', 'output_layer', 'linear', 'kernel', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "linear", "key_type": 2}, {"key": "kernel", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768, 1024]}}, "('params', 'output_layer', 'norm_final', 'scale', 'value')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layer", "key_type": 2}, {"key": "norm_final", "key_type": 2}, {"key": "scale", "key_type": 2}, {"key": "value", "key_type": 2}], "value_metadata": {"value_type": "jax.Array", "skip_deserialize": false, "write_shape": [768]}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null} \ No newline at end of file diff --git a/tinystories/uniform/checkpoints_ema/500000/state/_sharding b/tinystories/uniform/checkpoints_ema/500000/state/_sharding new file mode 100644 index 0000000000000000000000000000000000000000..2ced0ba60ec2ffa96c352791431a42482d422ef7 --- /dev/null +++ b/tinystories/uniform/checkpoints_ema/500000/state/_sharding @@ -0,0 +1 @@ +{"cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLm91dHB1dF9sYXllci5ub3JtX2ZpbmFsLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy40LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy41LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy42LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy43LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy44LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45Lm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy45LmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4wLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMC5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hZGFMTl9tb2R1bGF0aW9uLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX291dC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5hdHRuX3Frdi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMxLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5tbHBfZmMyLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4xMS5ub3JtMi5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4yLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm1scF9mYzIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLm5vcm0yLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fb3V0LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmF0dG5fcWt2LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24uYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmJsb2Nrcy4zLmFkYUxOX21vZHVsYXRpb24ua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [null], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnNpZ21hX21hcC5tbHBfMi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}","cGFyYW1zLmNvbmRpdGlvbmVyLnZvY2FiX2VtYmVkLmVtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [1], \"axis_names\": [\"fsdp\"], \"axis_types\": [\"AxisType.Auto\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [{\"id\": 0}]}}"} \ No newline at end of file diff --git a/tinystories/uniform/checkpoints_ema/500000/state/array_metadatas/process_0 b/tinystories/uniform/checkpoints_ema/500000/state/array_metadatas/process_0 new file mode 100644 index 0000000000000000000000000000000000000000..b4d02a076b7c3291b176ccf495f0ca8f8bcc852b --- /dev/null +++ b/tinystories/uniform/checkpoints_ema/500000/state/array_metadatas/process_0 @@ -0,0 +1 @@ +{"array_metadatas": [{"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.0.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.1.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.2.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.3.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.4.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.5.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.6.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.7.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.8.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.9.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.10.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.bias.value", "write_shape": [4608], "chunk_shape": [4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.adaLN_modulation.kernel.value", "write_shape": [128, 4608], "chunk_shape": [128, 4608], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_out.value", "write_shape": [12, 64, 768], "chunk_shape": [12, 64, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.attn_qkv.value", "write_shape": [768, 12, 3, 64], "chunk_shape": [768, 12, 3, 64], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.bias.value", "write_shape": [3072], "chunk_shape": [3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc1.kernel.value", "write_shape": [768, 3072], "chunk_shape": [768, 3072], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.bias.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.mlp_fc2.kernel.value", "write_shape": [3072, 768], "chunk_shape": [3072, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm1.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.blocks.11.norm2.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_0.kernel.value", "write_shape": [256, 128], "chunk_shape": [256, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.bias.value", "write_shape": [128], "chunk_shape": [128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.sigma_map.mlp_2.kernel.value", "write_shape": [128, 128], "chunk_shape": [128, 128], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.conditioner.vocab_embed.embedding.value", "write_shape": [1026, 768], "chunk_shape": [1026, 768], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.bias.value", "write_shape": [1536], "chunk_shape": [1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.adaLN_modulation.kernel.value", "write_shape": [128, 1536], "chunk_shape": [128, 1536], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.bias.value", "write_shape": [1024], "chunk_shape": [1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.linear.kernel.value", "write_shape": [768, 1024], "chunk_shape": [768, 1024], "ext_metadata": null}}, {"array_metadata": {"param_name": "params.output_layer.norm_final.scale.value", "write_shape": [768], "chunk_shape": [768], "ext_metadata": null}}]} \ No newline at end of file diff --git a/tinystories/uniform/checkpoints_ema/500000/state/d/910c50af0dc291d00467420cb49c6b8e b/tinystories/uniform/checkpoints_ema/500000/state/d/910c50af0dc291d00467420cb49c6b8e new file mode 100644 index 0000000000000000000000000000000000000000..edd5fa17a92e1cf792b6aea4c77dd12277c00f42 Binary files /dev/null and b/tinystories/uniform/checkpoints_ema/500000/state/d/910c50af0dc291d00467420cb49c6b8e differ diff --git a/tinystories/uniform/checkpoints_ema/500000/state/manifest.ocdbt b/tinystories/uniform/checkpoints_ema/500000/state/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..42d596e772cfe413cfd352e2bbcd7b02ac987ab0 Binary files /dev/null and b/tinystories/uniform/checkpoints_ema/500000/state/manifest.ocdbt differ diff --git a/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/1c6efc97c23cc5d2340fcd3d860bd810 b/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/1c6efc97c23cc5d2340fcd3d860bd810 new file mode 100644 index 0000000000000000000000000000000000000000..79c6812fa69506d2ed7349af6080ac8b9c1f7718 Binary files /dev/null and b/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/1c6efc97c23cc5d2340fcd3d860bd810 differ diff --git a/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/b31984a4a9b00362b5d6511619176a90 b/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/b31984a4a9b00362b5d6511619176a90 new file mode 100644 index 0000000000000000000000000000000000000000..aa19dde01ce322884805871a873432eca5047fa1 Binary files /dev/null and b/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/b31984a4a9b00362b5d6511619176a90 differ diff --git a/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/ea92c267744755209c1853785346aad1 b/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/ea92c267744755209c1853785346aad1 new file mode 100644 index 0000000000000000000000000000000000000000..27be22e12b3fc2ffd513217c2c4cef4c02420241 --- /dev/null +++ b/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/ea92c267744755209c1853785346aad1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e847b8dc12d2afcb3f7aeec654e74ec02da94fabebf88b91dc237151f01d548 +size 2186748 diff --git a/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/f8169566ee0804773f27cf30054bc094 b/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/f8169566ee0804773f27cf30054bc094 new file mode 100644 index 0000000000000000000000000000000000000000..ca2b6489c1a99e92233d5a114c7abfad0df77427 --- /dev/null +++ b/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/d/f8169566ee0804773f27cf30054bc094 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6c31aff677c29e90ae75041397335fe572d6a72fc688df3b29f67337addb848 +size 346218496 diff --git a/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt b/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt new file mode 100644 index 0000000000000000000000000000000000000000..1aedd0e198a5d08933ac34dabcd3daa20655164a Binary files /dev/null and b/tinystories/uniform/checkpoints_ema/500000/state/ocdbt.process_0/manifest.ocdbt differ