QuantaSparkLabs commited on
Commit
ec71667
·
verified ·
1 Parent(s): d4d1738

Delete checkpoint-550/tokenizer.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. checkpoint-550/tokenizer.json +0 -87
checkpoint-550/tokenizer.json DELETED
@@ -1,87 +0,0 @@
1
- {
2
- "version": "1.0",
3
- "truncation": {
4
- "direction": "Right",
5
- "max_length": 4,
6
- "strategy": "LongestFirst",
7
- "stride": 0
8
- },
9
- "padding": null,
10
- "added_tokens": [
11
- {
12
- "id": 0,
13
- "content": "<|endoftext|>",
14
- "single_word": false,
15
- "lstrip": false,
16
- "rstrip": false,
17
- "normalized": false,
18
- "special": true
19
- },
20
- {
21
- "id": 1,
22
- "content": "<|pad|>",
23
- "single_word": false,
24
- "lstrip": false,
25
- "rstrip": false,
26
- "normalized": false,
27
- "special": true
28
- }
29
- ],
30
- "normalizer": null,
31
- "pre_tokenizer": {
32
- "type": "ByteLevel",
33
- "add_prefix_space": true,
34
- "trim_offsets": true,
35
- "use_regex": true
36
- },
37
- "post_processor": {
38
- "type": "TemplateProcessing",
39
- "single": [
40
- {
41
- "Sequence": {
42
- "id": "A",
43
- "type_id": 0
44
- }
45
- }
46
- ],
47
- "pair": [
48
- {
49
- "Sequence": {
50
- "id": "A",
51
- "type_id": 0
52
- }
53
- },
54
- {
55
- "Sequence": {
56
- "id": "B",
57
- "type_id": 1
58
- }
59
- }
60
- ],
61
- "special_tokens": {}
62
- },
63
- "decoder": {
64
- "type": "ByteLevel",
65
- "add_prefix_space": true,
66
- "trim_offsets": true,
67
- "use_regex": true
68
- },
69
- "model": {
70
- "type": "BPE",
71
- "dropout": null,
72
- "unk_token": null,
73
- "continuing_subword_prefix": null,
74
- "end_of_word_suffix": null,
75
- "fuse_unk": false,
76
- "byte_fallback": false,
77
- "ignore_merges": false,
78
- "vocab": {
79
- "<|endoftext|>": 0,
80
- "<|pad|>": 1,
81
- "a": 2,
82
- "b": 3,
83
- "Ġ": 4
84
- },
85
- "merges": []
86
- }
87
- }