FHxw / config.json
aixk's picture
Upload folder using huggingface_hub
6861f23 verified
Raw History Blame Contribute Delete
26.3 kB
{ "hf_repo_id": "aixk/FHxw", "hf_dataset_repo_id": "aixk/chronosense-data-zz", "hf_vocab_dataset": "aixk/mdb6", "local_checkpoint_dir": "./checkpoints_titanmax_zz", "local_dataset_cache_dir": "./cached_dataset_titanmax_zz", "local_tokenizer_cache_dir": "./cached_tokenizer_titanmax_zz", "web_export_dir": "./vlite_export_titanmax_zz", "max_steps": 15000, "batch_size": 8, "accum_steps": 4, "log_interval": 10, "val_interval": 250, "keep_last_n_ckpts": 2, "learning_rate": 0.0006, "min_lr": 1.5e-05, "warmup_steps": 300, "decay_start_ratio": 0.85, "weight_decay": 0.05, "max_seq_len": 512, "vocab_size": 16384, "concept_size": 8192, "hidden_dim": 768, "num_layers": 12, "num_heads": 12, "head_dim": 64, "intermediate_dim": 2048, "chunk_size": 64, "conv_k": 4, "attention_layers": [ 3, 7, 11 ], "label_smoothing": 0.05, "logit_soft_capping": 30.0, "num_folds": 16, "vlite_quant_mode": "bitnet_1.58b", "architectures": [ "FHN_FS9_TitanMax_zz" ], "model_type": "fhn_titanmax_zz_knowledge_engine", "vlite_format": "vlite_bitnet_1.58b_packed", "vlite_layers": { "blocks.0.w_mdot.weight": { "orig_shape": [ 1, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.0.w_mdot.weight.packed", "scale_key": "blocks.0.w_mdot.weight.scale" }, "blocks.0.n1_conv.weight": { "orig_shape": [ 768, 1, 4 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.0.n1_conv.weight.packed", "scale_key": "blocks.0.n1_conv.weight.scale" }, "blocks.0.q_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.0.q_proj.weight.packed", "scale_key": "blocks.0.q_proj.weight.scale" }, "blocks.0.k_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.0.k_proj.weight.packed", "scale_key": "blocks.0.k_proj.weight.scale" }, "blocks.0.v_proj.weight": { "orig_shape": [ 384, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.0.v_proj.weight.packed", "scale_key": "blocks.0.v_proj.weight.scale" }, "blocks.0.decay_proj.weight": { "orig_shape": [ 12, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.0.decay_proj.weight.packed", "scale_key": "blocks.0.decay_proj.weight.scale" }, "blocks.0.out_proj.weight": { "orig_shape": [ 768, 384 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.0.out_proj.weight.packed", "scale_key": "blocks.0.out_proj.weight.scale" }, "blocks.0.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.0.ffn.w1.weight.packed", "scale_key": "blocks.0.ffn.w1.weight.scale" }, "blocks.0.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.0.ffn.w2.weight.packed", "scale_key": "blocks.0.ffn.w2.weight.scale" }, "blocks.0.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.0.ffn.w3.weight.packed", "scale_key": "blocks.0.ffn.w3.weight.scale" }, "blocks.1.w_mdot.weight": { "orig_shape": [ 1, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.1.w_mdot.weight.packed", "scale_key": "blocks.1.w_mdot.weight.scale" }, "blocks.1.n1_conv.weight": { "orig_shape": [ 768, 1, 4 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.1.n1_conv.weight.packed", "scale_key": "blocks.1.n1_conv.weight.scale" }, "blocks.1.q_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.1.q_proj.weight.packed", "scale_key": "blocks.1.q_proj.weight.scale" }, "blocks.1.k_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.1.k_proj.weight.packed", "scale_key": "blocks.1.k_proj.weight.scale" }, "blocks.1.v_proj.weight": { "orig_shape": [ 384, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.1.v_proj.weight.packed", "scale_key": "blocks.1.v_proj.weight.scale" }, "blocks.1.decay_proj.weight": { "orig_shape": [ 12, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.1.decay_proj.weight.packed", "scale_key": "blocks.1.decay_proj.weight.scale" }, "blocks.1.out_proj.weight": { "orig_shape": [ 768, 384 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.1.out_proj.weight.packed", "scale_key": "blocks.1.out_proj.weight.scale" }, "blocks.1.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.1.ffn.w1.weight.packed", "scale_key": "blocks.1.ffn.w1.weight.scale" }, "blocks.1.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.1.ffn.w2.weight.packed", "scale_key": "blocks.1.ffn.w2.weight.scale" }, "blocks.1.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.1.ffn.w3.weight.packed", "scale_key": "blocks.1.ffn.w3.weight.scale" }, "blocks.2.w_mdot.weight": { "orig_shape": [ 1, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.2.w_mdot.weight.packed", "scale_key": "blocks.2.w_mdot.weight.scale" }, "blocks.2.n1_conv.weight": { "orig_shape": [ 768, 1, 4 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.2.n1_conv.weight.packed", "scale_key": "blocks.2.n1_conv.weight.scale" }, "blocks.2.q_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.2.q_proj.weight.packed", "scale_key": "blocks.2.q_proj.weight.scale" }, "blocks.2.k_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.2.k_proj.weight.packed", "scale_key": "blocks.2.k_proj.weight.scale" }, "blocks.2.v_proj.weight": { "orig_shape": [ 384, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.2.v_proj.weight.packed", "scale_key": "blocks.2.v_proj.weight.scale" }, "blocks.2.decay_proj.weight": { "orig_shape": [ 12, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.2.decay_proj.weight.packed", "scale_key": "blocks.2.decay_proj.weight.scale" }, "blocks.2.out_proj.weight": { "orig_shape": [ 768, 384 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.2.out_proj.weight.packed", "scale_key": "blocks.2.out_proj.weight.scale" }, "blocks.2.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.2.ffn.w1.weight.packed", "scale_key": "blocks.2.ffn.w1.weight.scale" }, "blocks.2.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.2.ffn.w2.weight.packed", "scale_key": "blocks.2.ffn.w2.weight.scale" }, "blocks.2.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.2.ffn.w3.weight.packed", "scale_key": "blocks.2.ffn.w3.weight.scale" }, "blocks.3.qkv.weight": { "orig_shape": [ 2304, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.3.qkv.weight.packed", "scale_key": "blocks.3.qkv.weight.scale" }, "blocks.3.out_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.3.out_proj.weight.packed", "scale_key": "blocks.3.out_proj.weight.scale" }, "blocks.3.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.3.ffn.w1.weight.packed", "scale_key": "blocks.3.ffn.w1.weight.scale" }, "blocks.3.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.3.ffn.w2.weight.packed", "scale_key": "blocks.3.ffn.w2.weight.scale" }, "blocks.3.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.3.ffn.w3.weight.packed", "scale_key": "blocks.3.ffn.w3.weight.scale" }, "blocks.4.w_mdot.weight": { "orig_shape": [ 1, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.4.w_mdot.weight.packed", "scale_key": "blocks.4.w_mdot.weight.scale" }, "blocks.4.n1_conv.weight": { "orig_shape": [ 768, 1, 4 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.4.n1_conv.weight.packed", "scale_key": "blocks.4.n1_conv.weight.scale" }, "blocks.4.q_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.4.q_proj.weight.packed", "scale_key": "blocks.4.q_proj.weight.scale" }, "blocks.4.k_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.4.k_proj.weight.packed", "scale_key": "blocks.4.k_proj.weight.scale" }, "blocks.4.v_proj.weight": { "orig_shape": [ 384, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.4.v_proj.weight.packed", "scale_key": "blocks.4.v_proj.weight.scale" }, "blocks.4.decay_proj.weight": { "orig_shape": [ 12, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.4.decay_proj.weight.packed", "scale_key": "blocks.4.decay_proj.weight.scale" }, "blocks.4.out_proj.weight": { "orig_shape": [ 768, 384 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.4.out_proj.weight.packed", "scale_key": "blocks.4.out_proj.weight.scale" }, "blocks.4.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.4.ffn.w1.weight.packed", "scale_key": "blocks.4.ffn.w1.weight.scale" }, "blocks.4.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.4.ffn.w2.weight.packed", "scale_key": "blocks.4.ffn.w2.weight.scale" }, "blocks.4.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.4.ffn.w3.weight.packed", "scale_key": "blocks.4.ffn.w3.weight.scale" }, "blocks.5.w_mdot.weight": { "orig_shape": [ 1, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.5.w_mdot.weight.packed", "scale_key": "blocks.5.w_mdot.weight.scale" }, "blocks.5.n1_conv.weight": { "orig_shape": [ 768, 1, 4 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.5.n1_conv.weight.packed", "scale_key": "blocks.5.n1_conv.weight.scale" }, "blocks.5.q_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.5.q_proj.weight.packed", "scale_key": "blocks.5.q_proj.weight.scale" }, "blocks.5.k_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.5.k_proj.weight.packed", "scale_key": "blocks.5.k_proj.weight.scale" }, "blocks.5.v_proj.weight": { "orig_shape": [ 384, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.5.v_proj.weight.packed", "scale_key": "blocks.5.v_proj.weight.scale" }, "blocks.5.decay_proj.weight": { "orig_shape": [ 12, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.5.decay_proj.weight.packed", "scale_key": "blocks.5.decay_proj.weight.scale" }, "blocks.5.out_proj.weight": { "orig_shape": [ 768, 384 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.5.out_proj.weight.packed", "scale_key": "blocks.5.out_proj.weight.scale" }, "blocks.5.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.5.ffn.w1.weight.packed", "scale_key": "blocks.5.ffn.w1.weight.scale" }, "blocks.5.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.5.ffn.w2.weight.packed", "scale_key": "blocks.5.ffn.w2.weight.scale" }, "blocks.5.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.5.ffn.w3.weight.packed", "scale_key": "blocks.5.ffn.w3.weight.scale" }, "blocks.6.w_mdot.weight": { "orig_shape": [ 1, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.6.w_mdot.weight.packed", "scale_key": "blocks.6.w_mdot.weight.scale" }, "blocks.6.n1_conv.weight": { "orig_shape": [ 768, 1, 4 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.6.n1_conv.weight.packed", "scale_key": "blocks.6.n1_conv.weight.scale" }, "blocks.6.q_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.6.q_proj.weight.packed", "scale_key": "blocks.6.q_proj.weight.scale" }, "blocks.6.k_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.6.k_proj.weight.packed", "scale_key": "blocks.6.k_proj.weight.scale" }, "blocks.6.v_proj.weight": { "orig_shape": [ 384, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.6.v_proj.weight.packed", "scale_key": "blocks.6.v_proj.weight.scale" }, "blocks.6.decay_proj.weight": { "orig_shape": [ 12, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.6.decay_proj.weight.packed", "scale_key": "blocks.6.decay_proj.weight.scale" }, "blocks.6.out_proj.weight": { "orig_shape": [ 768, 384 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.6.out_proj.weight.packed", "scale_key": "blocks.6.out_proj.weight.scale" }, "blocks.6.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.6.ffn.w1.weight.packed", "scale_key": "blocks.6.ffn.w1.weight.scale" }, "blocks.6.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.6.ffn.w2.weight.packed", "scale_key": "blocks.6.ffn.w2.weight.scale" }, "blocks.6.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.6.ffn.w3.weight.packed", "scale_key": "blocks.6.ffn.w3.weight.scale" }, "blocks.7.qkv.weight": { "orig_shape": [ 2304, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.7.qkv.weight.packed", "scale_key": "blocks.7.qkv.weight.scale" }, "blocks.7.out_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.7.out_proj.weight.packed", "scale_key": "blocks.7.out_proj.weight.scale" }, "blocks.7.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.7.ffn.w1.weight.packed", "scale_key": "blocks.7.ffn.w1.weight.scale" }, "blocks.7.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.7.ffn.w2.weight.packed", "scale_key": "blocks.7.ffn.w2.weight.scale" }, "blocks.7.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.7.ffn.w3.weight.packed", "scale_key": "blocks.7.ffn.w3.weight.scale" }, "blocks.8.w_mdot.weight": { "orig_shape": [ 1, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.8.w_mdot.weight.packed", "scale_key": "blocks.8.w_mdot.weight.scale" }, "blocks.8.n1_conv.weight": { "orig_shape": [ 768, 1, 4 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.8.n1_conv.weight.packed", "scale_key": "blocks.8.n1_conv.weight.scale" }, "blocks.8.q_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.8.q_proj.weight.packed", "scale_key": "blocks.8.q_proj.weight.scale" }, "blocks.8.k_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.8.k_proj.weight.packed", "scale_key": "blocks.8.k_proj.weight.scale" }, "blocks.8.v_proj.weight": { "orig_shape": [ 384, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.8.v_proj.weight.packed", "scale_key": "blocks.8.v_proj.weight.scale" }, "blocks.8.decay_proj.weight": { "orig_shape": [ 12, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.8.decay_proj.weight.packed", "scale_key": "blocks.8.decay_proj.weight.scale" }, "blocks.8.out_proj.weight": { "orig_shape": [ 768, 384 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.8.out_proj.weight.packed", "scale_key": "blocks.8.out_proj.weight.scale" }, "blocks.8.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.8.ffn.w1.weight.packed", "scale_key": "blocks.8.ffn.w1.weight.scale" }, "blocks.8.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.8.ffn.w2.weight.packed", "scale_key": "blocks.8.ffn.w2.weight.scale" }, "blocks.8.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.8.ffn.w3.weight.packed", "scale_key": "blocks.8.ffn.w3.weight.scale" }, "blocks.9.w_mdot.weight": { "orig_shape": [ 1, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.9.w_mdot.weight.packed", "scale_key": "blocks.9.w_mdot.weight.scale" }, "blocks.9.n1_conv.weight": { "orig_shape": [ 768, 1, 4 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.9.n1_conv.weight.packed", "scale_key": "blocks.9.n1_conv.weight.scale" }, "blocks.9.q_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.9.q_proj.weight.packed", "scale_key": "blocks.9.q_proj.weight.scale" }, "blocks.9.k_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.9.k_proj.weight.packed", "scale_key": "blocks.9.k_proj.weight.scale" }, "blocks.9.v_proj.weight": { "orig_shape": [ 384, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.9.v_proj.weight.packed", "scale_key": "blocks.9.v_proj.weight.scale" }, "blocks.9.decay_proj.weight": { "orig_shape": [ 12, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.9.decay_proj.weight.packed", "scale_key": "blocks.9.decay_proj.weight.scale" }, "blocks.9.out_proj.weight": { "orig_shape": [ 768, 384 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.9.out_proj.weight.packed", "scale_key": "blocks.9.out_proj.weight.scale" }, "blocks.9.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.9.ffn.w1.weight.packed", "scale_key": "blocks.9.ffn.w1.weight.scale" }, "blocks.9.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.9.ffn.w2.weight.packed", "scale_key": "blocks.9.ffn.w2.weight.scale" }, "blocks.9.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.9.ffn.w3.weight.packed", "scale_key": "blocks.9.ffn.w3.weight.scale" }, "blocks.10.w_mdot.weight": { "orig_shape": [ 1, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.10.w_mdot.weight.packed", "scale_key": "blocks.10.w_mdot.weight.scale" }, "blocks.10.n1_conv.weight": { "orig_shape": [ 768, 1, 4 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.10.n1_conv.weight.packed", "scale_key": "blocks.10.n1_conv.weight.scale" }, "blocks.10.q_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.10.q_proj.weight.packed", "scale_key": "blocks.10.q_proj.weight.scale" }, "blocks.10.k_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.10.k_proj.weight.packed", "scale_key": "blocks.10.k_proj.weight.scale" }, "blocks.10.v_proj.weight": { "orig_shape": [ 384, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.10.v_proj.weight.packed", "scale_key": "blocks.10.v_proj.weight.scale" }, "blocks.10.decay_proj.weight": { "orig_shape": [ 12, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.10.decay_proj.weight.packed", "scale_key": "blocks.10.decay_proj.weight.scale" }, "blocks.10.out_proj.weight": { "orig_shape": [ 768, 384 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.10.out_proj.weight.packed", "scale_key": "blocks.10.out_proj.weight.scale" }, "blocks.10.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.10.ffn.w1.weight.packed", "scale_key": "blocks.10.ffn.w1.weight.scale" }, "blocks.10.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.10.ffn.w2.weight.packed", "scale_key": "blocks.10.ffn.w2.weight.scale" }, "blocks.10.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.10.ffn.w3.weight.packed", "scale_key": "blocks.10.ffn.w3.weight.scale" }, "blocks.11.qkv.weight": { "orig_shape": [ 2304, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.11.qkv.weight.packed", "scale_key": "blocks.11.qkv.weight.scale" }, "blocks.11.out_proj.weight": { "orig_shape": [ 768, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.11.out_proj.weight.packed", "scale_key": "blocks.11.out_proj.weight.scale" }, "blocks.11.ffn.w1.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.11.ffn.w1.weight.packed", "scale_key": "blocks.11.ffn.w1.weight.scale" }, "blocks.11.ffn.w2.weight": { "orig_shape": [ 2048, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.11.ffn.w2.weight.packed", "scale_key": "blocks.11.ffn.w2.weight.scale" }, "blocks.11.ffn.w3.weight": { "orig_shape": [ 768, 2048 ], "quant_mode": "bitnet_1.58b", "packed_key": "blocks.11.ffn.w3.weight.packed", "scale_key": "blocks.11.ffn.w3.weight.scale" }, "lm_head.weight": { "orig_shape": [ 16384, 768 ], "quant_mode": "bitnet_1.58b", "packed_key": "lm_head.weight.packed", "scale_key": "lm_head.weight.scale" } }, "torch_dtype": "float16" }
{
"hf_repo_id": "aixk/FHxw",
"hf_dataset_repo_id": "aixk/chronosense-data-zz",
"hf_vocab_dataset": "aixk/mdb6",
"local_checkpoint_dir": "./checkpoints_titanmax_zz",
"local_dataset_cache_dir": "./cached_dataset_titanmax_zz",
"local_tokenizer_cache_dir": "./cached_tokenizer_titanmax_zz",
"web_export_dir": "./vlite_export_titanmax_zz",
"max_steps": 15000,
"batch_size": 8,
"accum_steps": 4,
"log_interval": 10,
"val_interval": 250,
"keep_last_n_ckpts": 2,
"learning_rate": 0.0006,
"min_lr": 1.5e-05,
"warmup_steps": 300,
"decay_start_ratio": 0.85,
"weight_decay": 0.05,
"max_seq_len": 512,
"vocab_size": 16384,
"concept_size": 8192,
"hidden_dim": 768,
"num_layers": 12,
"num_heads": 12,
"head_dim": 64,
"intermediate_dim": 2048,
"chunk_size": 64,
"conv_k": 4,
"attention_layers": [
3,
7,
11
],
"label_smoothing": 0.05,
"logit_soft_capping": 30.0,
"num_folds": 16,
"vlite_quant_mode": "bitnet_1.58b",
"architectures": [
"FHN_FS9_TitanMax_zz"
],
"model_type": "fhn_titanmax_zz_knowledge_engine",
"vlite_format": "vlite_bitnet_1.58b_packed",
"vlite_layers": {
"blocks.0.w_mdot.weight": {
"orig_shape": [
1,
768
],
"quant_mode": "bitnet_1.58b",
"packed_key": "blocks.0.w_mdot.weight.packed",
"scale_key": "blocks.0.w_mdot.weight.scale"
},
"blocks.0.n1_conv.weight": {
"orig_shape": [
768,
1,
4
],
"quant_mode": "bitnet_1.58b",
"packed_key": "blocks.0.n1_conv.weight.packed",
"scale_key": "blocks.0.n1_conv.weight.scale"
},
"blocks.0.q_proj.weight": {
"orig_shape": [
768,
768
],
"quant_mode": "bitnet_1.58b",
"packed_key": "blocks.0.q_proj.weight.packed",
"scale_key": "blocks.0.q_proj.weight.scale"
},
"blocks.0.k_proj.weight": {
"orig_shape": [
768,
768
],
"quant_mode": "bitnet_1.58b",
"packed_key": "blocks.0.k_proj.weight.packed",
"scale_key": "blocks.0.k_proj.weight.scale"
},
"blocks.0.v_proj.weight": {
"orig_shape": [
384,
768
],
"quant_mode": "bitnet_1.58b",
"packed_key": "blocks.0.v_proj.weight.packed",
"scale_key": "blocks.0.v_proj.weight.scale"
},
"blocks.0.decay_proj.weight": {
"orig_shape": [
12,
768
],
"quant_mode": "bitnet_1.58b",
"packed_key": "blocks.0.decay_proj.weight.packed",
"scale_key": "blocks.0.decay_proj.weight.scale"
},
"blocks.0.out_proj.weight": {