Hugging Face's logo

anthonym21
/
json-tokenizer-structured

Text Generation
Transformers
English
tokenizer
json
bpe
structured-data
llm
Model card Files
xet
Community
json-tokenizer-structured
File size: 1,619 Bytes
cdef25b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
{
  "version": "json-tokenizer-v1",
  "bpe_vocab_size": 4096,
  "max_key_vocab": 512,
  "min_key_freq": 2,
  "bpe_min_freq": 2,
  "key_vocab": {
    "type": 32,
    "name": 33,
    "path": 34,
    "timestamp": 35,
    "geometry": 36,
    "coordinates": 37,
    "properties": 38,
    "id": 39,
    "country_code": 40,
    "admin_region": 41,
    "population": 42,
    "elevation_m": 43,
    "timezone": 44,
    "feature_class": 45,
    "feature_code": 46,
    "last_updated": 47,
    "level": 48,
    "service": 49,
    "trace_id": 50,
    "span_id": 51,
    "method": 52,
    "status_code": 53,
    "duration_ms": 54,
    "request_size": 55,
    "response_size": 56,
    "user_id": 57,
    "region": 58,
    "value": 59,
    "app": 60,
    "metadata": 61,
    "labels": 62,
    "spec": 63,
    "cpu": 64,
    "memory": 65,
    "httpGet": 66,
    "port": 67,
    "initialDelaySeconds": 68,
    "periodSeconds": 69,
    "apiVersion": 70,
    "kind": 71,
    "namespace": 72,
    "version": 73,
    "tier": 74,
    "annotations": 75,
    "deployment.kubernetes.io/revision": 76,
    "replicas": 77,
    "selector": 78,
    "matchLabels": 79,
    "template": 80,
    "containers": 81,
    "image": 82,
    "ports": 83,
    "containerPort": 84,
    "protocol": 85,
    "resources": 86,
    "requests": 87,
    "limits": 88,
    "env": 89,
    "livenessProbe": 90,
    "readinessProbe": 91,
    "imagePullSecrets": 92,
    "from": 93,
    "contributor": 94,
    "chat_format": 95,
    "conversations": 96
  },
  "key_offset": 32,
  "bpe_offset": 97
}