Hugging Face's logo

philipjohnbasile
/
wisp-coder-110m

Text Generation
Transformers
Safetensors
MLX
code
llama
fill-in-the-middle
multi-token-prediction
speculative-decoding
apple-silicon
text-generation-inference
Model card Files
xet
 Community
wisp-coder-110m
File size: 2,875 Bytes
818282c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
{
  "beta1": 0.9,
  "beta2": 0.95,
  "bias_correction": true,
  "ckpt_interval": 100,
  "compute_dtype": "bfloat16",
  "data_dir": "data/shards",
  "data_index": "data/shards/index.json",
  "data_integrity": {
    "receipt": "config/run1_shard_integrity_receipt.json",
    "role": "current",
    "sha256": "5831ecd4a471fbe07e19b212bc3de44bed0b0b6b456083e888f66802937bf471"
  },
  "dim": 768,
  "eps": 1e-08,
  "eval_interval": 500,
  "ffn_hidden": 2048,
  "fim_chunk": 1024,
  "fim_rate": 0.7,
  "grad_accum": 16,
  "grad_clip": 1.0,
  "log_interval": 10,
  "lr": 0.0006,
  "lr_min": 6e-05,
  "max_steps": 19073,
  "micro_batch": 8,
  "mtp_depth": 2,
  "mtp_layers": 1,
  "mtp_weight": 0.3,
  "n_heads": 12,
  "n_kv_heads": 4,
  "n_layers": 12,
  "norm_eps": 1e-05,
  "out_dir": "out/run1",
  "rope_theta": 100000.0,
  "run_name": "wisp-run1-110m-code",
  "sampler_reset_steps": [
    300
  ],
  "seed": 1337,
  "seq_len": 2048,
  "snapshot_interval": 2000,
  "sources": [
    {
      "data_dir": "python",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.24
    },
    {
      "data_dir": "javascript",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.13
    },
    {
      "data_dir": "typescript",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.12
    },
    {
      "data_dir": "go",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.09
    },
    {
      "data_dir": "rust",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.09
    },
    {
      "data_dir": "java",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.08
    },
    {
      "data_dir": "c",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.05
    },
    {
      "data_dir": "shell",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.03
    },
    {
      "data_dir": "sql",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.02
    },
    {
      "data_dir": "markdown",
      "quality_gate": true,
      "repo": "bigcode/starcoderdata",
      "text_field": "content",
      "weight": 0.07
    },
    {
      "name": "sample-10BT",
      "quality_gate": true,
      "repo": "HuggingFaceFW/fineweb-edu",
      "text_field": "text",
      "weight": 0.08
    }
  ],
  "tie_embeddings": true,
  "tokenizer_path": "tokenizer/code32k.json",
  "vocab_size": 32768,
  "warmup_steps": 1000,
  "weight_decay": 0.1
}