Hugging Face's logo

Jessiesj
/
script_gpt2-story

Text Generation
Transformers
Safetensors
gpt2
text-generation-inference
Model card Files
xet
Community
Jessiesj commited on
Commit
931d2dd
·
verified ·
1 Parent(s): 23230e8

Upload tokenizer

Browse files
Files changed (6) hide show
  1. added_tokens.json +16 -0
  2. merges.txt +0 -0
  3. special_tokens_map.json +46 -0
  4. tokenizer.json +0 -0
  5. tokenizer_config.json +156 -0
  6. vocab.json +0 -0
added_tokens.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</action>": 50260,
3
+ "</character>": 50262,
4
+ "</date>": 50268,
5
+ "</meta>": 50266,
6
+ "</note>": 50264,
7
+ "</parenthetical>": 50270,
8
+ "</scene>": 50258,
9
+ "<action>": 50259,
10
+ "<character>": 50261,
11
+ "<date>": 50267,
12
+ "<meta>": 50265,
13
+ "<note>": 50263,
14
+ "<parenthetical>": 50269,
15
+ "<scene>": 50257
16
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<scene>",
4
+ "</scene>",
5
+ "<action>",
6
+ "</action>",
7
+ "<character>",
8
+ "</character>",
9
+ "<note>",
10
+ "</note>",
11
+ "<meta>",
12
+ "</meta>",
13
+ "<date>",
14
+ "</date>",
15
+ "<parenthetical>",
16
+ "</parenthetical>"
17
+ ],
18
+ "bos_token": {
19
+ "content": "<|endoftext|>",
20
+ "lstrip": false,
21
+ "normalized": true,
22
+ "rstrip": false,
23
+ "single_word": false
24
+ },
25
+ "eos_token": {
26
+ "content": "<|endoftext|>",
27
+ "lstrip": false,
28
+ "normalized": true,
29
+ "rstrip": false,
30
+ "single_word": false
31
+ },
32
+ "pad_token": {
33
+ "content": "<|endoftext|>",
34
+ "lstrip": false,
35
+ "normalized": true,
36
+ "rstrip": false,
37
+ "single_word": false
38
+ },
39
+ "unk_token": {
40
+ "content": "<|endoftext|>",
41
+ "lstrip": false,
42
+ "normalized": true,
43
+ "rstrip": false,
44
+ "single_word": false
45
+ }
46
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "50256": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "50257": {
13
+ "content": "<scene>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "50258": {
21
+ "content": "</scene>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "50259": {
29
+ "content": "<action>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "50260": {
37
+ "content": "</action>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "50261": {
45
+ "content": "<character>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "50262": {
53
+ "content": "</character>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "50263": {
61
+ "content": "<note>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "50264": {
69
+ "content": "</note>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "50265": {
77
+ "content": "<meta>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "50266": {
85
+ "content": "</meta>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "50267": {
93
+ "content": "<date>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "50268": {
101
+ "content": "</date>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "50269": {
109
+ "content": "<parenthetical>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "50270": {
117
+ "content": "</parenthetical>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ }
124
+ },
125
+ "additional_special_tokens": [
126
+ "<scene>",
127
+ "</scene>",
128
+ "<action>",
129
+ "</action>",
130
+ "<character>",
131
+ "</character>",
132
+ "<note>",
133
+ "</note>",
134
+ "<meta>",
135
+ "</meta>",
136
+ "<date>",
137
+ "</date>",
138
+ "<parenthetical>",
139
+ "</parenthetical>"
140
+ ],
141
+ "bos_token": "<|endoftext|>",
142
+ "clean_up_tokenization_spaces": false,
143
+ "eos_token": "<|endoftext|>",
144
+ "extra_special_tokens": {},
145
+ "max_length": 512,
146
+ "model_max_length": 1024,
147
+ "pad_to_multiple_of": null,
148
+ "pad_token": "<|endoftext|>",
149
+ "pad_token_type_id": 0,
150
+ "padding_side": "right",
151
+ "stride": 0,
152
+ "tokenizer_class": "GPT2Tokenizer",
153
+ "truncation_side": "right",
154
+ "truncation_strategy": "longest_first",
155
+ "unk_token": "<|endoftext|>"
156
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff