Hugging Face's logo

SystemAdmin123
/
opt-350m

Text Generation
Transformers
Safetensors
opt
axolotl
Generated from Trainer
conversational
text-generation-inference
Model card Files
xet
 Community
SystemAdmin123 commited on
Commit
5d2ca86
·
verified ·
1 Parent(s): 5b5a1f5

Training in progress, step 20

Browse files
Files changed (9) hide show
  1. axolotl_config.yaml +51 -0
  2. config.json +31 -0
  3. merges.txt +0 -0
  4. model.safetensors +3 -0
  5. special_tokens_map.json +30 -0
  6. tokenizer.json +0 -0
  7. tokenizer_config.json +33 -0
  8. training_args.bin +3 -0
  9. vocab.json +0 -0
axolotl_config.yaml ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ base_model: facebook/opt-350m
2
+ batch_size: 96
3
+ bf16: true
4
+ chat_template: tokenizer_default_fallback_alpaca
5
+ datasets:
6
+ - format: custom
7
+ path: argilla/databricks-dolly-15k-curated-en
8
+ type:
9
+ field_input: original-instruction
10
+ field_instruction: original-instruction
11
+ field_output: original-response
12
+ format: '{instruction} {input}'
13
+ no_input_format: '{instruction}'
14
+ system_format: '{system}'
15
+ system_prompt: ''
16
+ device_map: auto
17
+ eval_sample_packing: false
18
+ eval_steps: 20
19
+ flash_attention: true
20
+ gradient_checkpointing: true
21
+ group_by_length: true
22
+ hub_model_id: SystemAdmin123/opt-350m
23
+ hub_strategy: checkpoint
24
+ learning_rate: 0.0002
25
+ logging_steps: 10
26
+ lr_scheduler: cosine
27
+ max_steps: 10000
28
+ micro_batch_size: 24
29
+ model_type: AutoModelForCausalLM
30
+ num_epochs: 100
31
+ optimizer: adamw_bnb_8bit
32
+ output_dir: /root/.sn56/axolotl/tmp/opt-350m
33
+ pad_to_sequence_len: true
34
+ resize_token_embeddings_to_32x: false
35
+ sample_packing: true
36
+ save_steps: 20
37
+ save_total_limit: 1
38
+ sequence_len: 2048
39
+ tokenizer_type: GPT2TokenizerFast
40
+ torch_dtype: bf16
41
+ training_args_kwargs:
42
+ hub_private_repo: true
43
+ trust_remote_code: true
44
+ val_set_size: 0.1
45
+ wandb_entity: ''
46
+ wandb_mode: online
47
+ wandb_name: facebook/opt-350m-argilla/databricks-dolly-15k-curated-en
48
+ wandb_project: Gradients-On-Demand
49
+ wandb_run: your_name
50
+ wandb_runid: default
51
+ warmup_ratio: 0.05
config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "facebook/opt-350m",
3
+ "_remove_final_layer_norm": false,
4
+ "activation_dropout": 0.0,
5
+ "activation_function": "relu",
6
+ "architectures": [
7
+ "OPTForCausalLM"
8
+ ],
9
+ "attention_dropout": 0.0,
10
+ "bos_token_id": 2,
11
+ "do_layer_norm_before": false,
12
+ "dropout": 0.1,
13
+ "enable_bias": true,
14
+ "eos_token_id": 2,
15
+ "ffn_dim": 4096,
16
+ "hidden_size": 1024,
17
+ "init_std": 0.02,
18
+ "layer_norm_elementwise_affine": true,
19
+ "layerdrop": 0.0,
20
+ "max_position_embeddings": 2048,
21
+ "model_type": "opt",
22
+ "num_attention_heads": 16,
23
+ "num_hidden_layers": 24,
24
+ "pad_token_id": 1,
25
+ "prefix": "</s>",
26
+ "torch_dtype": "bfloat16",
27
+ "transformers_version": "4.48.1",
28
+ "use_cache": false,
29
+ "vocab_size": 50265,
30
+ "word_embed_proj_dim": 512
31
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5abfbc4ca39f5a130360788b6dd8970c52cecc1e11057fba387e8decc03bb7e9
3
+ size 662430992
special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "</s>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<pad>",
18
+ "lstrip": false,
19
+ "normalized": true,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "</s>",
25
+ "lstrip": false,
26
+ "normalized": true,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "1": {
6
+ "content": "<pad>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "2": {
14
+ "content": "</s>",
15
+ "lstrip": false,
16
+ "normalized": true,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ }
21
+ },
22
+ "bos_token": "</s>",
23
+ "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '### Instruction: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ '### Response: ' + message['content'] + eos_token}}{% endif %}{% endfor %}",
24
+ "clean_up_tokenization_spaces": false,
25
+ "eos_token": "</s>",
26
+ "errors": "replace",
27
+ "extra_special_tokens": {},
28
+ "model_max_length": 1000000000000000019884624838656,
29
+ "pad_token": "<pad>",
30
+ "tokenizer_class": "GPT2Tokenizer",
31
+ "unk_token": "</s>",
32
+ "use_fast": true
33
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38c45a65d2feaa5c8b7363d20e9b9067a7c858a6c5c36582490c8414ec321027
3
+ size 6840
vocab.json ADDED
The diff for this file is too large to render. See raw diff