File size: 1,012 Bytes
54b7249 6d2e1d0 54b7249 6d2e1d0 54b7249 6d2e1d0 54b7249 6d2e1d0 54b7249 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | {
"backbone_freeze_steps": 0,
"clip_grad_per_group": false,
"combine_mhc": true,
"compressed_vocab_size": 21091,
"conv_dilation": 3,
"conv_kernel_size": 4,
"conv_zero_init": true,
"embedding_dim": 512,
"enable_telemetry": false,
"enable_tokenizer_compression": true,
"engram_dtype": null,
"engram_version": "1.2.2",
"engram_vocab_size_per_ngram": [
128000,
128000
],
"entropy_loss_weight": 0.0,
"gating_zero_init": true,
"hc_mult": 4,
"hidden_size": 2048,
"layer_container_path": "model.layers",
"learning_rate_multiplier": 5.0,
"max_ngram_size": 3,
"model_type": "engram",
"n_head_per_ngram": 8,
"ngram_sizes": [
2,
3
],
"pad_id": 2,
"seed": 0,
"target_layers": [
2
],
"target_modules": null,
"tokenizer_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
"train_mode": null,
"transformers_version": null,
"use_cache": false,
"use_sparse_embeddings": true,
"weight_decay": 0.0,
"wrap_peft": false
}
|