Hugging Face's logo

madatnlp
/
torch-trinity

Text Generation
Transformers
PyTorch
TensorBoard
gpt2
text-generation-inference
Model card Files
xet
Metrics Community
1
madatnlp commited on
Commit
368ba0f
·
1 Parent(s): 76b7bd5

add tokenizer

Browse files
Files changed (12) hide show
  1. .gitignore +1 -0
  2. merges.txt +0 -0
  3. runs/Jun03_06-28-14_aiffel-ps-koco/1654237700.221692/events.out.tfevents.1654237700.aiffel-ps-koco.11916.1 +3 -0
  4. runs/Jun03_06-28-14_aiffel-ps-koco/events.out.tfevents.1654237700.aiffel-ps-koco.11916.0 +3 -0
  5. runs/Jun03_06-40-19_aiffel-ps-koco/1654238423.0772452/events.out.tfevents.1654238423.aiffel-ps-koco.15239.1 +3 -0
  6. runs/Jun03_06-40-19_aiffel-ps-koco/events.out.tfevents.1654238423.aiffel-ps-koco.15239.0 +3 -0
  7. runs/Jun03_06-56-38_aiffel-ps-koco/1654239401.576065/events.out.tfevents.1654239401.aiffel-ps-koco.19699.1 +3 -0
  8. runs/Jun03_06-56-38_aiffel-ps-koco/events.out.tfevents.1654239401.aiffel-ps-koco.19699.0 +3 -0
  9. special_tokens_map.json +1 -0
  10. tokenizer.json +0 -0
  11. tokenizer_config.json +1 -0
  12. vocab.json +0 -0
.gitignore ADDED
@@ -0,0 +1 @@
 
 
1
+ checkpoint-*/
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
runs/Jun03_06-28-14_aiffel-ps-koco/1654237700.221692/events.out.tfevents.1654237700.aiffel-ps-koco.11916.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cdedf0d487fc4d2977408613b5e096363a1d133bace8b6b296f8fb5afa4b2470
3
+ size 5161
runs/Jun03_06-28-14_aiffel-ps-koco/events.out.tfevents.1654237700.aiffel-ps-koco.11916.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49fd0763ad74257d144b9fbcaf217c6b2b4dd1316f3babbfd43b471b242161ea
3
+ size 15151
runs/Jun03_06-40-19_aiffel-ps-koco/1654238423.0772452/events.out.tfevents.1654238423.aiffel-ps-koco.15239.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a36262e8f39633c3b32b8fe7179f5b3c55161d6c3838274744405eeb78a27e3
3
+ size 5161
runs/Jun03_06-40-19_aiffel-ps-koco/events.out.tfevents.1654238423.aiffel-ps-koco.15239.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fb935c5b5177d003602540324d91d2199490245e93e5a632a332b59eea94437
3
+ size 21300
runs/Jun03_06-56-38_aiffel-ps-koco/1654239401.576065/events.out.tfevents.1654239401.aiffel-ps-koco.19699.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c4a985a1e38a964b5adad6081e29684d2772afedb70051e27fc00d958a5bd34
3
+ size 5161
runs/Jun03_06-56-38_aiffel-ps-koco/events.out.tfevents.1654239401.aiffel-ps-koco.19699.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed93866db493228c5c565d655462ec48493649f2d3f1ee013fddd2cb99fe6543
3
+ size 6137
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": "<mask>"}
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"unk_token": "<|endoftext|>", "bos_token": "<s>", "eos_token": "</s>", "add_prefix_space": true, "special_tokens_map_file": "/home/tov324/.cache/huggingface/transformers/c2ab65b9d700d0871fd407d489869d7b93f69fb5f1a58fb1fac796fd43b9ea27.1f5b09bb43973b9fbd2ba75c9fe44ffab036b980c4e6a9d779aa7707913416fe", "name_or_path": "skt/ko-gpt-trinity-1.2B-v0.5", "tokenizer_class": "GPT2Tokenizer"}
vocab.json ADDED
The diff for this file is too large to render. See raw diff