Hugging Face's logo

huggingface-mic
/
Inkling-mic

Safetensors
inkling_mm_model
mic
test-artifact
Model card Files
xet
 Community
Inkling-mic / build.json
tarekziade's picture
tarekziade HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
06ce3d5 verified
Raw History Blame Contribute Delete
10.4 kB
{
"fixture": "huggingface-mic/Inkling-mic",
"key": "209ad3f38c33094dcca1347ed1ad26a7598cb9864074d672dec9bc1efd6b52a8",
"key_fields": {
"architectures": [
"InklingForConditionalGeneration"
],
"output_hidden_states": false,
"return_dict": true,
"chunk_size_feed_forward": 0,
"is_encoder_decoder": false,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1"
},
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1
},
"problem_type": null,
"text_config": {
"architectures": null,
"output_hidden_states": false,
"return_dict": true,
"dtype": null,
"chunk_size_feed_forward": 0,
"is_encoder_decoder": false,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1"
},
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1
},
"problem_type": null,
"vocab_size": 201024,
"unpadded_vocab_size": 200058,
"hidden_size": 64,
"num_hidden_layers": 3,
"num_attention_heads": 4,
"num_key_value_heads": 2,
"head_dim": 16,
"swa_num_attention_heads": 2,
"swa_num_key_value_heads": 2,
"swa_head_dim": 16,
"sliding_window_size": 512,
"d_rel": 16,
"rel_extent": 1024,
"log_scaling_n_floor": 128000,
"log_scaling_alpha": 0.1,
"local_layer_ids": [
0,
1,
2,
3,
4,
6,
7,
8,
9,
10,
12,
13,
14,
15,
16,
18,
19,
20,
21,
22,
24,
25,
26,
27,
28,
30,
31,
32,
33,
34,
36,
37,
38,
39,
40,
42,
43,
44,
45,
46,
48,
49,
50,
51,
52,
54,
55,
56,
57,
58,
60,
61,
62,
63,
64
],
"layer_types": [
"hybrid_sliding",
"hybrid_sliding",
"hybrid"
],
"max_position_embeddings": 1048576,
"rms_norm_eps": 1e-06,
"conv_kernel_size": 4,
"mlp_layer_types": [
"dense",
"sparse",
"sparse"
],
"intermediate_size": 128,
"hidden_act": "silu",
"moe_intermediate_size": 32,
"n_routed_experts": 4,
"num_experts_per_tok": 2,
"n_shared_experts": 2,
"shared_expert_sink": true,
"route_scale": 8.0,
"logits_mup_width_multiplier": 24.0,
"rms_norm_eps_moe_gate": 1e-06,
"attention_dropout": 0.0,
"initializer_range": 0.02,
"pad_token_id": null,
"bos_token_id": 1,
"eos_token_id": 2,
"num_mtp_layers": 8,
"chain_hidden_post_norm": false,
"mtp_hidden_states_first": true,
"mtp_local_layer_ids": [
0,
2,
4,
5,
6,
7
],
"number_of_conv_states": 4,
"_name_or_path": "",
"q_bias": false,
"o_bias": false,
"use_embed_norm": true,
"use_sconv": true,
"final_logit_softcapping": null,
"use_gate_bias": true,
"gate_activation": "sigmoid",
"norm_after_topk": true,
"use_global_scale": true,
"model_type": "inkling_text",
"output_attentions": false
},
"audio_config": {
"architectures": null,
"output_hidden_states": false,
"return_dict": true,
"dtype": null,
"chunk_size_feed_forward": 0,
"is_encoder_decoder": false,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1"
},
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1
},
"problem_type": null,
"n_mel_bins": 80,
"mel_vocab_size": 16,
"text_hidden_size": 64,
"rms_norm_eps": 1e-06,
"initializer_range": 0.02,
"_name_or_path": "",
"decoder_dmodel": 6144,
"bias": false,
"dmel_min_value": -7.0,
"dmel_max_value": 2.0,
"use_audio_norm": true,
"audio_mode": "dmel",
"model_type": "inkling_audio",
"output_attentions": false
},
"vision_config": {
"architectures": null,
"output_hidden_states": false,
"return_dict": true,
"dtype": null,
"chunk_size_feed_forward": 0,
"is_encoder_decoder": false,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1"
},
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1
},
"problem_type": null,
"text_hidden_size": 64,
"patch_size": 40,
"temporal_patch_size": 2,
"num_channels": 3,
"hidden_size": 64,
"n_layers": 2,
"num_attention_heads": 4,
"rms_norm_eps": 1e-06,
"initializer_range": 0.02,
"_name_or_path": "",
"vision_encoder_type": "hmlp",
"decoder_dmodel": 6144,
"n_channels": 3,
"use_vision_norm": true,
"model_type": "inkling_vision",
"output_attentions": false
},
"image_token_id": 200054,
"audio_token_id": 200053,
"image_bos_token_id": 200005,
"audio_bos_token_id": 200020,
"model_type": "inkling_mm_model",
"eos_token_id": 200006,
"mtp_config": {
"num_nextn_predict_layers": 8,
"chain_hidden_post_norm": false,
"local_layer_ids": [
0,
2,
4,
5,
6,
7
]
},
"output_attentions": false
},
"covers": {
"thinkingmachines/Inkling": "828496eeae4c243ff1a22f7f28ff83694f2f7bc9"
},
"built_from": {
"repo": "thinkingmachines/Inkling",
"revision": "828496eeae4c243ff1a22f7f28ff83694f2f7bc9"
},
"seed": 17,
"dtype": "float32",
"builder": "example/build_fixtures.py",
"builder_revision": "e8034a51dc828ccfb8c6ad3d765dd2275181f0b8",
"recipe": {
"seed": 17,
"model_class": "InklingForConditionalGeneration"
},
"shrink": "rule",
"model_class": "InklingForConditionalGeneration",
"parameters": 26736419,
"missing_parameters": [],
"serialization": {
"file": "model.safetensors",
"reverted_conversions": [
"WeightRenaming: ['model\\\\.llm\\\\.layers'] -> ['model.language_model.layers']",
"WeightRenaming: ['model\\\\.llm\\\\.embed_norm\\\\.weight'] -> ['model.language_model.embed_tokens.embed_norm.weight']",
"WeightRenaming: ['model\\\\.llm\\\\.embed\\\\.weight'] -> ['model.language_model.embed_tokens.weight']",
"WeightRenaming: ['model\\\\.llm\\\\.norm\\\\.weight'] -> ['model.language_model.norm.weight']",
"WeightRenaming: ['model\\\\.llm\\\\.unembed\\\\.weight'] -> ['lm_head.weight']",
"WeightRenaming: ['model\\\\.audio\\\\.'] -> ['model.audio_tower.']",
"WeightRenaming: ['model\\\\.visual'] -> ['model.vision_tower']",
"WeightRenaming: ['vision_tower.layers.linear_(\\\\d+)'] -> ['vision_tower.encoder_layers.\\\\1.projection']",
"WeightRenaming: ['vision_tower.layers.norm_(\\\\d+)'] -> ['vision_tower.encoder_layers.\\\\1.layer_norm']",
"WeightRenaming: ['audio_tower.encoder.weight'] -> ['audio_tower.embed_audio_tokens.embed_audio_tokens.weight']",
"WeightRenaming: ['audio_tower.final_norm.weight'] -> ['audio_tower.norm.weight']",
"WeightConverter: ['shared_w13_weight'] -> ['gate_proj', 'up_proj']",
"WeightRenaming: ['shared_w2_weight'] -> ['down_proj']",
"WeightConverter: ['mlp.experts.w13_weight'] -> ['mlp.experts.gate_up_proj']",
"WeightRenaming: ['mlp.experts.w2_weight'] -> ['mlp.experts.down_proj']",
"WeightConverter: ['mlp.w13_dn.weight'] -> ['mlp.gate_proj.weight', 'mlp.up_proj.weight']",
"WeightRenaming: ['mlp.w2_md.weight'] -> ['mlp.down_proj.weight']",
"WeightRenaming: ['mlp.gate.bias'] -> ['mlp.gate.e_score_correction_bias']",
"WeightRenaming: ['attn\\\\.wq_du'] -> ['self_attn.q_proj']",
"WeightRenaming: ['attn\\\\.wk_dv'] -> ['self_attn.k_proj']",
"WeightRenaming: ['attn\\\\.wv_dv'] -> ['self_attn.v_proj']",
"WeightRenaming: ['attn\\\\.wr_du'] -> ['self_attn.r_proj']",
"WeightRenaming: ['attn\\\\.wo_ud'] -> ['self_attn.o_proj']",
"WeightRenaming: ['\\\\.attn\\\\.q_norm'] -> ['.self_attn.q_norm']",
"WeightRenaming: ['\\\\.attn\\\\.k_norm'] -> ['.self_attn.k_norm']",
"WeightRenaming: ['\\\\.attn\\\\.k_sconv'] -> ['.self_attn.k_sconv.conv1d']",
"WeightRenaming: ['\\\\.attn\\\\.v_sconv'] -> ['.self_attn.v_sconv.conv1d']",
"WeightRenaming: ['\\\\.attn\\\\.rel_logits_proj'] -> ['.self_attn.rel_logits_proj']",
"WeightRenaming: ['attn_sconv\\\\.weight'] -> ['attn_sconv.conv1d.weight']",
"WeightRenaming: ['mlp_sconv\\\\.weight'] -> ['mlp_sconv.conv1d.weight']",
"WeightRenaming: ['mlp_norm'] -> ['post_attention_layernorm']",
"WeightRenaming: ['attn_norm'] -> ['input_layernorm']"
]
},
"tester_sizes": {
"source": "https://raw.githubusercontent.com/huggingface/transformers/main/tests/models/inkling/test_modeling_inkling.py",
"fields": {
"text_config.swa_head_dim": 16,
"text_config.swa_num_attention_heads": 2,
"text_config.swa_num_key_value_heads": 2
}
},
"checkpoint_inspection": "parameter headers/indexes; bounded binary fallback",
"numeric_health_scope": "CPU float32/bfloat16/float16; autocast for low-precision losses",
"captured": {
"call": "training step",
"inputs": [
"attention_mask",
"input_ids",
"labels"
],
"class": "InklingForConditionalGeneration"
},
"weights": "random",
"gate": {
"random": {
"pass": true,
"accepted_insensitive": {},
"threshold": 0.001,
"deltas": {
"input_ids:roll_batch": 0.05011682212352753,
"input_ids:reverse": 0.04614920914173126,
"attention_mask:all_ones": null
},
"weakest": "input_ids:reverse",
"no_perturbation_defined": [],
"not_a_forward_parameter": []
}
},
"numeric_health": {
"float32": {
"outputs_finite": true,
"loss_finite": true,
"gradients_finite": true,
"finite": true
},
"bfloat16": {
"outputs_finite": true,
"loss_finite": true,
"gradients_finite": true,
"finite": true
},
"float16": {
"outputs_finite": true,
"loss_finite": true,
"unscaled_gradients_finite": true,
"finite": true
}
},
"copied_unchanged": [
"chat_template.jinja",
"processor_config.json",
"special_tokens_map.json",
"tokenizer.json",
"tokenizer_config.json"
],
"environment": {
"python": "3.12.12",
"machine": "arm64",
"torch": "2.14.1",
"transformers": "5.22.0.dev0",
"tokenizers": "0.23.3"
},
"sha256": {
"chat_template.jinja": "0aa1aa0c729d90176dcaa00c440c8faffca2957ffb2cc4b79456ee6d02bcf43b",
"config.json": "967261d39b02fa42d82ca2dbd87b8bac1303db35a94ae43c8bb134694e91272e",
"generation_config.json": "f3186777071146682ed0debc54b80ef604c42bf6728a1f7b6fbe7cf39b796c3b",
"model.safetensors": "4787e37ad6b44b84e4403405e0f472b06b3bf681cbf7d162f16a9215870270a6",
"processor_config.json": "b4a3962ea5f7ec39f40b5cf14e57ce99776c3dcce4756a110f7a169809e3a04c",
"special_tokens_map.json": "abc97715b4b3b30eb65ea6895afd7b529c32bd10c28901f9ddae7edc39723b0f",
"tokenizer.json": "9fb6333a7db8fe5da90728e741e4a3ee4ac2ae12c5dd4958cc6f31688787d3c2",
"tokenizer_config.json": "2e36c9748a2081abb935b2e745ee22e82efa32589c2500df7e5bc0f93145cd77"
}
}