Upload folder using huggingface_hub

final/config.json

@@ -0,0 +1,100 @@
+{
+  "hidden_size": 896,
+  "num_hidden_layers": 16,
+  "intermediate_size": 256,
+  "vocab_size": 32000,
+  "num_attention_heads": 14,
+  "num_key_value_heads": 2,
+  "attention_type": "gqa",
+  "attention_dropout": 0.0,
+  "use_qk_norm": true,
+  "sliding_window": null,
+  "causal_conv_kernel_size": 4,
+  "causal_conv_dilation_cycle": 8,
+  "causal_state_rank": 16,
+  "causal_context_gate_init": 1.0,
+  "causal_contextual_mix_init": 0.0,
+  "causal_context_fusion_size": 0,
+  "causal_stable_delta": false,
+  "causal_delta_chunk_size": 512,
+  "causal_delta_timescales": 1,
+  "causal_delta_collision_normalized": false,
+  "causal_delta_lexical_values": false,
+  "causal_delta_lexical_forge": false,
+  "causal_delta_occurrence_address": false,
+  "tr_mha_num_experts": 4,
+  "tr_mha_adapter_rank": 8,
+  "tr_mha_top_k": 2,
+  "tr_mha_adapter_gate_init": 0.1,
+  "tr_mha_id_primary_logit": 2.0,
+  "tr_mha_id_secondary_logit": 1.0,
+  "tr_mha_id_other_logit": -2.0,
+  "tr_mha_verifier_gate_init": 0.1,
+  "tr_mha_verifier_temperature": 1.0,
+  "tr_mha_targets": "qv",
+  "max_position_embeddings": 2048,
+  "rope_theta": 10000.0,
+  "rope_type": "standard",
+  "rope_fraction": 1.0,
+  "mlp_type": "tr_hash_engine",
+  "hidden_act": "silu",
+  "num_experts": 4,
+  "expert_initialization": "gpt_normal",
+  "token_frequencies": null,
+  "routing_strategy": "token_id_multi_hash",
+  "route_hash_count": 2,
+  "lsh_routing": false,
+  "lsh_bits": 0,
+  "lsh_from_layer": 0,
+  "lsh_threshold_mode": "zero",
+  "shared_expert": true,
+  "shared_intermediate_size": 3072,
+  "shared_expert_chunk_tokens": 0,
+  "use_shared_routed_gates": false,
+  "shared_gate_init": 1.0,
+  "routed_gate_init": 1.0,
+  "shared_output_scale": 1.0,
+  "routed_output_scale": 2.0,
+  "shared_output_scale_first_layer": null,
+  "shared_output_scale_last_layer": null,
+  "routed_output_scale_first_layer": null,
+  "routed_output_scale_last_layer": null,
+  "top_k": 2,
+  "top_k_primary_weight": 0.5,
+  "learn_hash_pair_gates": false,
+  "hash_pair_gate_init": 0.5,
+  "learn_hash_channel_modulation": false,
+  "hash_channel_scale_init": 0.0,
+  "static_expert_capacity": false,
+  "use_custom_kernels": "auto",
+  "collect_moe_telemetry": false,
+  "use_cggr": "auto",
+  "active_num_experts": null,
+  "active_expert_width": null,
+  "lexical_object_rank": 16,
+  "lexical_object_gate_init": 0.1,
+  "tie_lexical_object_embeddings": false,
+  "micro_num_experts": 4,
+  "micro_expert_width": 16,
+  "micro_expert_gate_init": 0.1,
+  "lexical_gqa_rank": 16,
+  "lexical_gqa_gate_init": 0.0,
+  "lexical_gqa_use_token_code": true,
+  "lexical_key_gate_init": 0.05,
+  "lexical_zipf_path": null,
+  "lexical_zipf_mode": "uniform",
+  "lexical_zipf_alpha": 0.25,
+  "lexical_zipf_floor": 0.1,
+  "lexical_zipf_permutation_seed": 1729,
+  "norm_type": "rmsnorm",
+  "norm_eps": 1e-06,
+  "tie_word_embeddings": true,
+  "use_sdpa": true,
+  "use_cache": true,
+  "initializer_range": 0.02,
+  "use_mup_init": false,
+  "use_mup_attn_scale": false,
+  "use_mup_output_mult": false,
+  "mup_base_width": 256,
+  "extra_config": {}
+}

final/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62a98cd7eae04b744089880379e7251d59c944883597aa78b301be10ce11f386
+size 813515200

final/model_config.yaml

@@ -0,0 +1,98 @@
+active_expert_width: null
+active_num_experts: null
+attention_dropout: 0.0
+attention_type: gqa
+causal_context_fusion_size: 0
+causal_context_gate_init: 1.0
+causal_contextual_mix_init: 0.0
+causal_conv_dilation_cycle: 8
+causal_conv_kernel_size: 4
+causal_delta_chunk_size: 512
+causal_delta_collision_normalized: false
+causal_delta_lexical_forge: false
+causal_delta_lexical_values: false
+causal_delta_occurrence_address: false
+causal_delta_timescales: 1
+causal_stable_delta: false
+causal_state_rank: 16
+collect_moe_telemetry: false
+expert_initialization: gpt_normal
+extra_config: {}
+hash_channel_scale_init: 0.0
+hash_pair_gate_init: 0.5
+hidden_act: silu
+hidden_size: 896
+initializer_range: 0.02
+intermediate_size: 256
+learn_hash_channel_modulation: false
+learn_hash_pair_gates: false
+lexical_gqa_gate_init: 0.0
+lexical_gqa_rank: 16
+lexical_gqa_use_token_code: true
+lexical_key_gate_init: 0.05
+lexical_object_gate_init: 0.1
+lexical_object_rank: 16
+lexical_zipf_alpha: 0.25
+lexical_zipf_floor: 0.1
+lexical_zipf_mode: uniform
+lexical_zipf_path: null
+lexical_zipf_permutation_seed: 1729
+lsh_bits: 0
+lsh_from_layer: 0
+lsh_routing: false
+lsh_threshold_mode: zero
+max_position_embeddings: 2048
+micro_expert_gate_init: 0.1
+micro_expert_width: 16
+micro_num_experts: 4
+mlp_type: tr_hash_engine
+mup_base_width: 256
+norm_eps: 1.0e-06
+norm_type: rmsnorm
+num_attention_heads: 14
+num_experts: 4
+num_hidden_layers: 16
+num_key_value_heads: 2
+rope_fraction: 1.0
+rope_theta: 10000.0
+rope_type: standard
+route_hash_count: 2
+routed_gate_init: 1.0
+routed_output_scale: 2.0
+routed_output_scale_first_layer: null
+routed_output_scale_last_layer: null
+routing_strategy: token_id_multi_hash
+shared_expert: true
+shared_expert_chunk_tokens: 0
+shared_gate_init: 1.0
+shared_intermediate_size: 3072
+shared_output_scale: 1.0
+shared_output_scale_first_layer: null
+shared_output_scale_last_layer: null
+sliding_window: null
+static_expert_capacity: false
+tie_lexical_object_embeddings: false
+tie_word_embeddings: true
+token_frequencies: null
+top_k: 2
+top_k_primary_weight: 0.5
+tr_mha_adapter_gate_init: 0.1
+tr_mha_adapter_rank: 8
+tr_mha_id_other_logit: -2.0
+tr_mha_id_primary_logit: 2.0
+tr_mha_id_secondary_logit: 1.0
+tr_mha_num_experts: 4
+tr_mha_targets: qv
+tr_mha_top_k: 2
+tr_mha_verifier_gate_init: 0.1
+tr_mha_verifier_temperature: 1.0
+use_cache: true
+use_cggr: auto
+use_custom_kernels: auto
+use_mup_attn_scale: false
+use_mup_init: false
+use_mup_output_mult: false
+use_qk_norm: true
+use_sdpa: true
+use_shared_routed_gates: false
+vocab_size: 32000