Hugging Face's logo

etherluminifer
/
llm-from-scratch

Model card Files
xet
 Community
llm-from-scratch
File size: 786 Bytes
8c2e86c
 
 
 
39a09e9
8c2e86c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39a09e9
 
8c2e86c
39a09e9
 
8c2e86c
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
{
  "model": {
    "embedding_config": {
      "context_size": 1024,
      "vocab_size": 32000,
      "d_model": 768,
      "dropout_prob": 0.1
    },
    "transformer_config": {
      "mha_config": {
        "d_model": 768,
        "num_heads": 12,
        "dropout_prob": 0.1
      },
      "ff_expansion_factor": 4,
      "activation": "Gelu",
      "dropout_prob": 0.1
    },
    "num_transformer_layers": 12
  },
  "optimizer": {
    "grad_clipping": null,
    "beta_1": 0.9,
    "beta_2": 0.999,
    "epsilon": 0.00001,
    "weight_decay": 0.0001,
    "cautious_weight_decay": false,
    "amsgrad": false
  },
  "tokenizer_kind": "bpe",
  "tokenizer_path": "tokenizer.json",
  "num_epochs": 200,
  "batch_size": 6,
  "num_workers": 6,
  "seed": 123456,
  "learning_rate": 0.0001
}