Hugging Face's logo

jfrick
/
climbmix-code-tokenizer

Model card Files
xet
 Community
climbmix-code-tokenizer
File size: 1,903 Bytes
2d8a612
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
{
  "eval_sha256": "71e377e1008755b83876ebab70f509df46974387d270ca218ceb7a7263ade543",
  "tokenizers": {
    "climbmix-code-v1": {
      "tokenizer_sha256": "48aa6fb6d01d04067570637ca8a5ecced608f5df5585f9ce44a12c05c5281ae5",
      "sources": {
        "climbmix": {
          "bytes": 14743727,
          "tokens": 3217386,
          "roundtrip_failures": 0,
          "bytes_per_token": 4.582517298204194
        },
        "code_curated": {
          "bytes": 3682462,
          "tokens": 1023424,
          "roundtrip_failures": 0,
          "bytes_per_token": 3.598178272153086
        },
        "code_mit": {
          "bytes": 1733957,
          "tokens": 503413,
          "roundtrip_failures": 0,
          "bytes_per_token": 3.4444025084771352
        }
      },
      "digits": [
        "1",
        "2",
        "3",
        "4",
        "5",
        "\u0120+",
        "\u0120",
        "6",
        "7",
        "8",
        "\u0120=",
        "\u0120",
        "1",
        "3",
        "0",
        "2",
        "3"
      ]
    },
    "structured-code-v1": {
      "tokenizer_sha256": "add5b2c7542e4efded329735825ecdcb94df1432617c3d3e4b7508aa459432f9",
      "sources": {
        "climbmix": {
          "bytes": 14743727,
          "tokens": 4022037,
          "roundtrip_failures": 0,
          "bytes_per_token": 3.6657362923314727
        },
        "code_curated": {
          "bytes": 3682462,
          "tokens": 1046393,
          "roundtrip_failures": 0,
          "bytes_per_token": 3.519195942633408
        },
        "code_mit": {
          "bytes": 1733957,
          "tokens": 514537,
          "roundtrip_failures": 0,
          "bytes_per_token": 3.3699364671539658
        }
      },
      "digits": [
        "123",
        "45",
        "\u0120+",
        "\u0120",
        "678",
        "\u0120=",
        "\u0120",
        "130",
        "23"
      ]
    }
  }
}