tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
d03ef8c verified | { | |
| "fixture": "huggingface-mic/Jamba-tiny-dev-mic", | |
| "key": "ce507b862c67085517bdb80699420885f7ba5b0af9beece93aa10ae531c15c80", | |
| "key_fields": { | |
| "architectures": [ | |
| "JambaForCausalLM" | |
| ], | |
| "output_hidden_states": false, | |
| "return_dict": true, | |
| "chunk_size_feed_forward": 0, | |
| "is_encoder_decoder": false, | |
| "id2label": { | |
| "0": "LABEL_0", | |
| "1": "LABEL_1" | |
| }, | |
| "label2id": { | |
| "LABEL_0": 0, | |
| "LABEL_1": 1 | |
| }, | |
| "problem_type": null, | |
| "vocab_size": 65536, | |
| "tie_word_embeddings": false, | |
| "hidden_size": 64, | |
| "intermediate_size": 128, | |
| "num_hidden_layers": 5, | |
| "num_attention_heads": 4, | |
| "num_key_value_heads": 2, | |
| "hidden_act": "silu", | |
| "initializer_range": 0.02, | |
| "rms_norm_eps": 1e-06, | |
| "use_cache": true, | |
| "output_router_logits": false, | |
| "router_aux_loss_coef": 0.001, | |
| "pad_token_id": 0, | |
| "bos_token_id": 1, | |
| "eos_token_id": 2, | |
| "max_position_embeddings": 262144, | |
| "attention_dropout": 0.0, | |
| "num_experts_per_tok": 2, | |
| "num_experts": 4, | |
| "expert_layer_period": 2, | |
| "expert_layer_offset": 1, | |
| "attn_layer_period": 8, | |
| "attn_layer_offset": 4, | |
| "use_mamba_kernels": true, | |
| "mamba_d_state": 16, | |
| "mamba_d_conv": 4, | |
| "mamba_expand": 2, | |
| "mamba_dt_rank": 32, | |
| "mamba_conv_bias": true, | |
| "mamba_proj_bias": false, | |
| "use_mambapy": false, | |
| "use_associative_scan": true, | |
| "model_type": "jamba", | |
| "num_logits_to_keep": 1, | |
| "sliding_window": null, | |
| "output_attentions": false, | |
| "dtype": "bfloat16" | |
| }, | |
| "covers": { | |
| "ai21labs/Jamba-tiny-dev": "ed303361004ac875426a61675edecf8e9d976882" | |
| }, | |
| "built_from": { | |
| "repo": "ai21labs/Jamba-tiny-dev", | |
| "revision": "ed303361004ac875426a61675edecf8e9d976882" | |
| }, | |
| "seed": 17, | |
| "dtype": "bfloat16", | |
| "builder": "example/build_fixtures.py", | |
| "builder_revision": "76cba244631520dc5e504be76e22bb73fcf16b8b", | |
| "recipe": { | |
| "seed": 17 | |
| }, | |
| "shrink": "rule", | |
| "model_class": "JambaForCausalLM", | |
| "parameters": 8831936, | |
| "missing_parameters": [], | |
| "checkpoint_inspection": "parameter headers/indexes; bounded binary fallback", | |
| "numeric_health_scope": "CPU float32/bfloat16/float16; autocast for low-precision losses", | |
| "captured": { | |
| "call": "training step", | |
| "inputs": [ | |
| "attention_mask", | |
| "input_ids", | |
| "labels" | |
| ], | |
| "class": "JambaForCausalLM" | |
| }, | |
| "weights": "random", | |
| "gate": { | |
| "random": { | |
| "pass": true, | |
| "accepted_insensitive": {}, | |
| "threshold": 0.001, | |
| "deltas": { | |
| "input_ids:roll_batch": 0.763671875, | |
| "input_ids:reverse": 1.17578125, | |
| "attention_mask:all_ones": null | |
| }, | |
| "weakest": "input_ids:roll_batch", | |
| "no_perturbation_defined": [], | |
| "not_a_forward_parameter": [] | |
| } | |
| }, | |
| "numeric_health": { | |
| "float32": { | |
| "outputs_finite": true, | |
| "loss_finite": true, | |
| "gradients_finite": true, | |
| "finite": true | |
| }, | |
| "bfloat16": { | |
| "outputs_finite": true, | |
| "loss_finite": true, | |
| "gradients_finite": true, | |
| "finite": true | |
| }, | |
| "float16": { | |
| "outputs_finite": true, | |
| "loss_finite": true, | |
| "unscaled_gradients_finite": true, | |
| "finite": true | |
| } | |
| }, | |
| "copied_unchanged": [ | |
| "generation_config.json", | |
| "special_tokens_map.json", | |
| "tokenizer.json", | |
| "tokenizer.model", | |
| "tokenizer_config.json" | |
| ], | |
| "environment": { | |
| "python": "3.12.12", | |
| "machine": "arm64", | |
| "torch": "2.14.1", | |
| "transformers": "5.22.0.dev0", | |
| "tokenizers": "0.23.3" | |
| }, | |
| "sha256": { | |
| "config.json": "ce9577f4c98802184f1e5d0b326cea67af4e19840f1d426004811771a54a72a7", | |
| "generation_config.json": "0dd850e9c007f3046febe16658a39b6caa96768682f7fc4c356861be6b456216", | |
| "model.safetensors": "26d26ea890188b3ce4cf8b1aaa843d141559cedd782f180f0085289b84130f5d", | |
| "special_tokens_map.json": "ff56aa3285bed3b10bd29029296c4b290536dc174e54ab9c15e7c4edfa180492", | |
| "tokenizer.json": "ce194086b50346971dabd71bba69b9d4c40b3a40c055977dc9a75da9d39080c8", | |
| "tokenizer.model": "8b0df4fb43262c452ef37061951a06df4c63ca191d02a60ea08f14428af24376", | |
| "tokenizer_config.json": "88bc9a4a7a7dd363d9f559e6e85bdeda3bb49d3e6725252c3b23c3383a0527a0" | |
| } | |
| } | |