Hugging Face's logo

flopml
/
mamba

Model card Files
xet
Community
mamba
File size: 633 Bytes 
1d49067
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 

{
  "dataset": {
    "remote_path": "flpelerin/tinystories-10k",
    "batch_size": 4,
    "seq_length": 512
  },

  "tokenizer": {
    "max_length": 32000
  },

  "wandb": {
    "entity": "florianpelerin110304",
    "project": "mamba-trainer-test",
    "name": "",
    "api_key": "860f8753998c6e6dc356914de07e8855aa2f9642"
  },

  "model": {
    "params": {
      "d_model": 768,
      "n_layer": 24,

      "tie_embeddings": "false"
    }
  },

  "trainer": {
    "learning_rate": 1e-3,
    "num_epochs": 4,

    "inference": {
      "frequency": 0,
      "n_predict": 256,
      "seed_text": "One day, a little girl"
    }
  }
}