Upload folder using huggingface_hub
Browse files- README.md +25 -0
- checkpoint-2/config.json +43 -0
- checkpoint-2/generation_config.json +6 -0
- checkpoint-2/model.safetensors +3 -0
- checkpoint-2/optimizer.pt +3 -0
- checkpoint-2/rng_state.pth +3 -0
- checkpoint-2/scaler.pt +3 -0
- checkpoint-2/scheduler.pt +3 -0
- checkpoint-2/tokenizer.json +0 -0
- checkpoint-2/tokenizer_config.json +12 -0
- checkpoint-2/trainer_state.json +33 -0
- checkpoint-2/training_args.bin +3 -0
- checkpoint-3/config.json +43 -0
- checkpoint-3/generation_config.json +6 -0
- checkpoint-3/model.safetensors +3 -0
- checkpoint-3/optimizer.pt +3 -0
- checkpoint-3/rng_state.pth +3 -0
- checkpoint-3/scaler.pt +3 -0
- checkpoint-3/scheduler.pt +3 -0
- checkpoint-3/tokenizer.json +0 -0
- checkpoint-3/tokenizer_config.json +12 -0
- checkpoint-3/trainer_state.json +33 -0
- checkpoint-3/training_args.bin +3 -0
- config.json +43 -0
- generation_config.json +6 -0
- model.safetensors +3 -0
- tokenizer.json +0 -0
- tokenizer_config.json +12 -0
README.md
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
base_model: gpt2-large
|
| 4 |
+
tags:
|
| 5 |
+
- admin
|
| 6 |
+
- sql
|
| 7 |
+
- chatbot
|
| 8 |
+
- gpt2
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
# Admin AI GPT-2
|
| 12 |
+
|
| 13 |
+
This model is a **fine-tuned version of GPT-2 Large** for:
|
| 14 |
+
|
| 15 |
+
- Admin system assistance
|
| 16 |
+
- Read-only SQL generation
|
| 17 |
+
- ERP / CMS support
|
| 18 |
+
|
| 19 |
+
## Rules
|
| 20 |
+
- Read-only
|
| 21 |
+
- No DELETE / UPDATE SQL
|
| 22 |
+
- Admin-focused answers
|
| 23 |
+
|
| 24 |
+
## Intended Use
|
| 25 |
+
Use via controlled backend (FastAPI / .NET).
|
checkpoint-2/config.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_function": "gelu_new",
|
| 3 |
+
"add_cross_attention": false,
|
| 4 |
+
"architectures": [
|
| 5 |
+
"GPT2LMHeadModel"
|
| 6 |
+
],
|
| 7 |
+
"attn_pdrop": 0.1,
|
| 8 |
+
"bos_token_id": 50256,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"embd_pdrop": 0.1,
|
| 11 |
+
"eos_token_id": 50256,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"layer_norm_epsilon": 1e-05,
|
| 14 |
+
"model_type": "gpt2",
|
| 15 |
+
"n_ctx": 1024,
|
| 16 |
+
"n_embd": 1024,
|
| 17 |
+
"n_head": 16,
|
| 18 |
+
"n_inner": null,
|
| 19 |
+
"n_layer": 24,
|
| 20 |
+
"n_positions": 1024,
|
| 21 |
+
"n_special": 0,
|
| 22 |
+
"pad_token_id": null,
|
| 23 |
+
"predict_special_tokens": true,
|
| 24 |
+
"reorder_and_upcast_attn": false,
|
| 25 |
+
"resid_pdrop": 0.1,
|
| 26 |
+
"scale_attn_by_inverse_layer_idx": false,
|
| 27 |
+
"scale_attn_weights": true,
|
| 28 |
+
"summary_activation": null,
|
| 29 |
+
"summary_first_dropout": 0.1,
|
| 30 |
+
"summary_proj_to_labels": true,
|
| 31 |
+
"summary_type": "cls_index",
|
| 32 |
+
"summary_use_proj": true,
|
| 33 |
+
"task_specific_params": {
|
| 34 |
+
"text-generation": {
|
| 35 |
+
"do_sample": true,
|
| 36 |
+
"max_length": 50
|
| 37 |
+
}
|
| 38 |
+
},
|
| 39 |
+
"tie_word_embeddings": true,
|
| 40 |
+
"transformers_version": "5.0.0",
|
| 41 |
+
"use_cache": false,
|
| 42 |
+
"vocab_size": 50257
|
| 43 |
+
}
|
checkpoint-2/generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 50256,
|
| 4 |
+
"eos_token_id": 50256,
|
| 5 |
+
"transformers_version": "5.0.0"
|
| 6 |
+
}
|
checkpoint-2/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1fe2c3aa64cf154c9c9a9d6349778291668f2bb46751fad3b97857d1fe0e43bc
|
| 3 |
+
size 1419322880
|
checkpoint-2/optimizer.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:824478bcc8ca32a1e904f9008334ca1a07e1aea381754b2ca1b62be0f761dee7
|
| 3 |
+
size 1832
|
checkpoint-2/rng_state.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:52b1c1cf3e1e68a17893e5cf3b54822aee4972bfa568d1ba991b39f592f7b67c
|
| 3 |
+
size 14244
|
checkpoint-2/scaler.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7e14c352a55e10c999ee72775811104f1f50e266cbb49dba1e848461353dcea6
|
| 3 |
+
size 988
|
checkpoint-2/scheduler.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5fd07864881583932ac6af628d414f8aa4c2fc94496bd06d7c56ab6b8f9464e0
|
| 3 |
+
size 1064
|
checkpoint-2/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoint-2/tokenizer_config.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"backend": "tokenizers",
|
| 4 |
+
"bos_token": "<|endoftext|>",
|
| 5 |
+
"eos_token": "<|endoftext|>",
|
| 6 |
+
"errors": "replace",
|
| 7 |
+
"is_local": false,
|
| 8 |
+
"model_max_length": 1024,
|
| 9 |
+
"pad_token": "<|endoftext|>",
|
| 10 |
+
"tokenizer_class": "GPT2Tokenizer",
|
| 11 |
+
"unk_token": "<|endoftext|>"
|
| 12 |
+
}
|
checkpoint-2/trainer_state.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"best_global_step": null,
|
| 3 |
+
"best_metric": null,
|
| 4 |
+
"best_model_checkpoint": null,
|
| 5 |
+
"epoch": 2.0,
|
| 6 |
+
"eval_steps": 500,
|
| 7 |
+
"global_step": 2,
|
| 8 |
+
"is_hyper_param_search": false,
|
| 9 |
+
"is_local_process_zero": true,
|
| 10 |
+
"is_world_process_zero": true,
|
| 11 |
+
"log_history": [],
|
| 12 |
+
"logging_steps": 10,
|
| 13 |
+
"max_steps": 3,
|
| 14 |
+
"num_input_tokens_seen": 0,
|
| 15 |
+
"num_train_epochs": 3,
|
| 16 |
+
"save_steps": 500,
|
| 17 |
+
"stateful_callbacks": {
|
| 18 |
+
"TrainerControl": {
|
| 19 |
+
"args": {
|
| 20 |
+
"should_epoch_stop": false,
|
| 21 |
+
"should_evaluate": false,
|
| 22 |
+
"should_log": false,
|
| 23 |
+
"should_save": true,
|
| 24 |
+
"should_training_stop": false
|
| 25 |
+
},
|
| 26 |
+
"attributes": {}
|
| 27 |
+
}
|
| 28 |
+
},
|
| 29 |
+
"total_flos": 533277351936.0,
|
| 30 |
+
"train_batch_size": 1,
|
| 31 |
+
"trial_name": null,
|
| 32 |
+
"trial_params": null
|
| 33 |
+
}
|
checkpoint-2/training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5e5695045f421c1226e4d845e1c9d3abfdaa906103d1229ec3c233fa06cb349f
|
| 3 |
+
size 4792
|
checkpoint-3/config.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_function": "gelu_new",
|
| 3 |
+
"add_cross_attention": false,
|
| 4 |
+
"architectures": [
|
| 5 |
+
"GPT2LMHeadModel"
|
| 6 |
+
],
|
| 7 |
+
"attn_pdrop": 0.1,
|
| 8 |
+
"bos_token_id": 50256,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"embd_pdrop": 0.1,
|
| 11 |
+
"eos_token_id": 50256,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"layer_norm_epsilon": 1e-05,
|
| 14 |
+
"model_type": "gpt2",
|
| 15 |
+
"n_ctx": 1024,
|
| 16 |
+
"n_embd": 1024,
|
| 17 |
+
"n_head": 16,
|
| 18 |
+
"n_inner": null,
|
| 19 |
+
"n_layer": 24,
|
| 20 |
+
"n_positions": 1024,
|
| 21 |
+
"n_special": 0,
|
| 22 |
+
"pad_token_id": null,
|
| 23 |
+
"predict_special_tokens": true,
|
| 24 |
+
"reorder_and_upcast_attn": false,
|
| 25 |
+
"resid_pdrop": 0.1,
|
| 26 |
+
"scale_attn_by_inverse_layer_idx": false,
|
| 27 |
+
"scale_attn_weights": true,
|
| 28 |
+
"summary_activation": null,
|
| 29 |
+
"summary_first_dropout": 0.1,
|
| 30 |
+
"summary_proj_to_labels": true,
|
| 31 |
+
"summary_type": "cls_index",
|
| 32 |
+
"summary_use_proj": true,
|
| 33 |
+
"task_specific_params": {
|
| 34 |
+
"text-generation": {
|
| 35 |
+
"do_sample": true,
|
| 36 |
+
"max_length": 50
|
| 37 |
+
}
|
| 38 |
+
},
|
| 39 |
+
"tie_word_embeddings": true,
|
| 40 |
+
"transformers_version": "5.0.0",
|
| 41 |
+
"use_cache": false,
|
| 42 |
+
"vocab_size": 50257
|
| 43 |
+
}
|
checkpoint-3/generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 50256,
|
| 4 |
+
"eos_token_id": 50256,
|
| 5 |
+
"transformers_version": "5.0.0"
|
| 6 |
+
}
|
checkpoint-3/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1fe2c3aa64cf154c9c9a9d6349778291668f2bb46751fad3b97857d1fe0e43bc
|
| 3 |
+
size 1419322880
|
checkpoint-3/optimizer.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:824478bcc8ca32a1e904f9008334ca1a07e1aea381754b2ca1b62be0f761dee7
|
| 3 |
+
size 1832
|
checkpoint-3/rng_state.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:09e5c4354e1af0c6d17a56a57d4023d2ee27d1ce0a2bb5704d38903d6e17b91f
|
| 3 |
+
size 14244
|
checkpoint-3/scaler.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9a56aef2eeef4e35205cce139c28416373971535279b2f997a2f7bb7006299b3
|
| 3 |
+
size 988
|
checkpoint-3/scheduler.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5fd07864881583932ac6af628d414f8aa4c2fc94496bd06d7c56ab6b8f9464e0
|
| 3 |
+
size 1064
|
checkpoint-3/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoint-3/tokenizer_config.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"backend": "tokenizers",
|
| 4 |
+
"bos_token": "<|endoftext|>",
|
| 5 |
+
"eos_token": "<|endoftext|>",
|
| 6 |
+
"errors": "replace",
|
| 7 |
+
"is_local": false,
|
| 8 |
+
"model_max_length": 1024,
|
| 9 |
+
"pad_token": "<|endoftext|>",
|
| 10 |
+
"tokenizer_class": "GPT2Tokenizer",
|
| 11 |
+
"unk_token": "<|endoftext|>"
|
| 12 |
+
}
|
checkpoint-3/trainer_state.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"best_global_step": null,
|
| 3 |
+
"best_metric": null,
|
| 4 |
+
"best_model_checkpoint": null,
|
| 5 |
+
"epoch": 3.0,
|
| 6 |
+
"eval_steps": 500,
|
| 7 |
+
"global_step": 3,
|
| 8 |
+
"is_hyper_param_search": false,
|
| 9 |
+
"is_local_process_zero": true,
|
| 10 |
+
"is_world_process_zero": true,
|
| 11 |
+
"log_history": [],
|
| 12 |
+
"logging_steps": 10,
|
| 13 |
+
"max_steps": 3,
|
| 14 |
+
"num_input_tokens_seen": 0,
|
| 15 |
+
"num_train_epochs": 3,
|
| 16 |
+
"save_steps": 500,
|
| 17 |
+
"stateful_callbacks": {
|
| 18 |
+
"TrainerControl": {
|
| 19 |
+
"args": {
|
| 20 |
+
"should_epoch_stop": false,
|
| 21 |
+
"should_evaluate": false,
|
| 22 |
+
"should_log": false,
|
| 23 |
+
"should_save": true,
|
| 24 |
+
"should_training_stop": true
|
| 25 |
+
},
|
| 26 |
+
"attributes": {}
|
| 27 |
+
}
|
| 28 |
+
},
|
| 29 |
+
"total_flos": 799916027904.0,
|
| 30 |
+
"train_batch_size": 1,
|
| 31 |
+
"trial_name": null,
|
| 32 |
+
"trial_params": null
|
| 33 |
+
}
|
checkpoint-3/training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5e5695045f421c1226e4d845e1c9d3abfdaa906103d1229ec3c233fa06cb349f
|
| 3 |
+
size 4792
|
config.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_function": "gelu_new",
|
| 3 |
+
"add_cross_attention": false,
|
| 4 |
+
"architectures": [
|
| 5 |
+
"GPT2LMHeadModel"
|
| 6 |
+
],
|
| 7 |
+
"attn_pdrop": 0.1,
|
| 8 |
+
"bos_token_id": 50256,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"embd_pdrop": 0.1,
|
| 11 |
+
"eos_token_id": 50256,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"layer_norm_epsilon": 1e-05,
|
| 14 |
+
"model_type": "gpt2",
|
| 15 |
+
"n_ctx": 1024,
|
| 16 |
+
"n_embd": 1024,
|
| 17 |
+
"n_head": 16,
|
| 18 |
+
"n_inner": null,
|
| 19 |
+
"n_layer": 24,
|
| 20 |
+
"n_positions": 1024,
|
| 21 |
+
"n_special": 0,
|
| 22 |
+
"pad_token_id": null,
|
| 23 |
+
"predict_special_tokens": true,
|
| 24 |
+
"reorder_and_upcast_attn": false,
|
| 25 |
+
"resid_pdrop": 0.1,
|
| 26 |
+
"scale_attn_by_inverse_layer_idx": false,
|
| 27 |
+
"scale_attn_weights": true,
|
| 28 |
+
"summary_activation": null,
|
| 29 |
+
"summary_first_dropout": 0.1,
|
| 30 |
+
"summary_proj_to_labels": true,
|
| 31 |
+
"summary_type": "cls_index",
|
| 32 |
+
"summary_use_proj": true,
|
| 33 |
+
"task_specific_params": {
|
| 34 |
+
"text-generation": {
|
| 35 |
+
"do_sample": true,
|
| 36 |
+
"max_length": 50
|
| 37 |
+
}
|
| 38 |
+
},
|
| 39 |
+
"tie_word_embeddings": true,
|
| 40 |
+
"transformers_version": "5.0.0",
|
| 41 |
+
"use_cache": false,
|
| 42 |
+
"vocab_size": 50257
|
| 43 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 50256,
|
| 4 |
+
"eos_token_id": 50256,
|
| 5 |
+
"transformers_version": "5.0.0"
|
| 6 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1fe2c3aa64cf154c9c9a9d6349778291668f2bb46751fad3b97857d1fe0e43bc
|
| 3 |
+
size 1419322880
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"backend": "tokenizers",
|
| 4 |
+
"bos_token": "<|endoftext|>",
|
| 5 |
+
"eos_token": "<|endoftext|>",
|
| 6 |
+
"errors": "replace",
|
| 7 |
+
"is_local": false,
|
| 8 |
+
"model_max_length": 1024,
|
| 9 |
+
"pad_token": "<|endoftext|>",
|
| 10 |
+
"tokenizer_class": "GPT2Tokenizer",
|
| 11 |
+
"unk_token": "<|endoftext|>"
|
| 12 |
+
}
|