Hugging Face's logo

Ba2han
/
experimental2

Text Generation
Transformers
qwen3
Generated from Trainer
unsloth
sft
trl
custom_code
Model card Files
xet
 Community
Ba2han commited on
Commit
40a3711
·
verified ·
1 Parent(s): 8d14f09

Training in progress, step 300

Browse files
Files changed (7) hide show
  1. README.md +58 -0
  2. config.json +90 -0
  3. generation_config.json +13 -0
  4. model.safetensors +3 -0
  5. tokenizer.json +0 -0
  6. tokenizer_config.json +163 -0
  7. training_args.bin +3 -0
README.md ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ model_name: experimental2
4
+ tags:
5
+ - generated_from_trainer
6
+ - unsloth
7
+ - trl
8
+ - sft
9
+ licence: license
10
+ ---
11
+
12
+ # Model Card for experimental2
13
+
14
+ This model is a fine-tuned version of [None](https://huggingface.co/None).
15
+ It has been trained using [TRL](https://github.com/huggingface/trl).
16
+
17
+ ## Quick start
18
+
19
+ ```python
20
+ from transformers import pipeline
21
+
22
+ question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
23
+ generator = pipeline("text-generation", model="Ba2han/experimental2", device="cuda")
24
+ output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
25
+ print(output["generated_text"])
26
+ ```
27
+
28
+ ## Training procedure
29
+
30
+ [<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/batuhan409/huggingface/runs/8fcww366)
31
+
32
+
33
+ This model was trained with SFT.
34
+
35
+ ### Framework versions
36
+
37
+ - TRL: 0.24.0
38
+ - Transformers: 5.6.2
39
+ - Pytorch: 2.10.0
40
+ - Datasets: 4.3.0
41
+ - Tokenizers: 0.22.2
42
+
43
+ ## Citations
44
+
45
+
46
+
47
+ Cite TRL as:
48
+
49
+ ```bibtex
50
+ @misc{vonwerra2022trl,
51
+ title = {{TRL: Transformer Reinforcement Learning}},
52
+ author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
53
+ year = 2020,
54
+ journal = {GitHub repository},
55
+ publisher = {GitHub},
56
+ howpublished = {\url{https://github.com/huggingface/trl}}
57
+ }
58
+ ```
config.json ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3CanonForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "auto_map": {
8
+ "AutoModelForCausalLM": "patch.Qwen3CanonForCausalLM"
9
+ },
10
+ "bos_token_id": 50030,
11
+ "dtype": "bfloat16",
12
+ "eos_token_id": 50031,
13
+ "head_dim": 64,
14
+ "hidden_act": "silu",
15
+ "hidden_size": 1024,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 4096,
18
+ "layer_types": [
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention",
41
+ "full_attention",
42
+ "full_attention",
43
+ "full_attention",
44
+ "full_attention",
45
+ "full_attention",
46
+ "full_attention",
47
+ "full_attention",
48
+ "full_attention",
49
+ "full_attention",
50
+ "full_attention",
51
+ "full_attention",
52
+ "full_attention",
53
+ "full_attention",
54
+ "full_attention",
55
+ "full_attention",
56
+ "full_attention",
57
+ "full_attention",
58
+ "full_attention",
59
+ "full_attention",
60
+ "full_attention"
61
+ ],
62
+ "max_position_embeddings": 8192,
63
+ "max_window_layers": 42,
64
+ "model_name": "test_checkpoint",
65
+ "model_type": "qwen3",
66
+ "num_attention_heads": 12,
67
+ "num_hidden_layers": 42,
68
+ "num_key_value_heads": 4,
69
+ "pad_token_id": 50034,
70
+ "qk_norm_freeze_affine": true,
71
+ "resid_lambda_init": 1.0,
72
+ "rms_norm_eps": 1e-06,
73
+ "rope_parameters": {
74
+ "rope_theta": 50000,
75
+ "rope_type": "default"
76
+ },
77
+ "sliding_window": null,
78
+ "softcap_divisor": 7.5,
79
+ "softcap_logits": true,
80
+ "softcap_scale": 23.0,
81
+ "softcap_shift": 5.0,
82
+ "tie_word_embeddings": true,
83
+ "transformers_version": "5.6.2",
84
+ "unsloth_version": "2026.4.8",
85
+ "use_cache": false,
86
+ "use_qk_norm_patch": true,
87
+ "use_sliding_window": false,
88
+ "vocab_size": 50048,
89
+ "x0_lambda_init": 0.1
90
+ }
generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50030,
4
+ "eos_token_id": [
5
+ 50031
6
+ ],
7
+ "max_length": 8192,
8
+ "output_attentions": false,
9
+ "output_hidden_states": false,
10
+ "pad_token_id": 50034,
11
+ "transformers_version": "5.6.2",
12
+ "use_cache": false
13
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bcda3dd42bd162aa2fed5bf0f413cf6ccd8edbd9e8b4685e2589d63f1519595f
3
+ size 1335861472
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|begin_of_text|>",
4
+ "clean_up_tokenization_spaces": true,
5
+ "eos_token": "<|end_of_text|>",
6
+ "is_local": true,
7
+ "local_files_only": false,
8
+ "model_input_names": [
9
+ "input_ids",
10
+ "attention_mask"
11
+ ],
12
+ "model_max_length": 8192,
13
+ "pad_token": "<|finetune_right_pad_id|>",
14
+ "padding_side": "right",
15
+ "tokenizer_class": "TokenizersBackend",
16
+ "unk_token": null,
17
+ "added_tokens_decoder": {
18
+ "50030": {
19
+ "content": "<|begin_of_text|>",
20
+ "single_word": false,
21
+ "lstrip": false,
22
+ "rstrip": false,
23
+ "normalized": false,
24
+ "special": true
25
+ },
26
+ "50031": {
27
+ "content": "<|end_of_text|>",
28
+ "single_word": false,
29
+ "lstrip": false,
30
+ "rstrip": false,
31
+ "normalized": false,
32
+ "special": true
33
+ },
34
+ "50032": {
35
+ "content": "<|reserved_special_token_0|>",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ },
42
+ "50033": {
43
+ "content": "<|reserved_special_token_1|>",
44
+ "single_word": false,
45
+ "lstrip": false,
46
+ "rstrip": false,
47
+ "normalized": false,
48
+ "special": true
49
+ },
50
+ "50034": {
51
+ "content": "<|finetune_right_pad_id|>",
52
+ "single_word": false,
53
+ "lstrip": false,
54
+ "rstrip": false,
55
+ "normalized": false,
56
+ "special": true
57
+ },
58
+ "50035": {
59
+ "content": "<|reserved_special_token_2|>",
60
+ "single_word": false,
61
+ "lstrip": false,
62
+ "rstrip": false,
63
+ "normalized": false,
64
+ "special": true
65
+ },
66
+ "50036": {
67
+ "content": "<|start_header_id|>",
68
+ "single_word": false,
69
+ "lstrip": false,
70
+ "rstrip": false,
71
+ "normalized": false,
72
+ "special": true
73
+ },
74
+ "50037": {
75
+ "content": "<|end_header_id|>",
76
+ "single_word": false,
77
+ "lstrip": false,
78
+ "rstrip": false,
79
+ "normalized": false,
80
+ "special": true
81
+ },
82
+ "50038": {
83
+ "content": "<|eom_id|>",
84
+ "single_word": false,
85
+ "lstrip": false,
86
+ "rstrip": false,
87
+ "normalized": false,
88
+ "special": true
89
+ },
90
+ "50039": {
91
+ "content": "<|eot_id|>",
92
+ "single_word": false,
93
+ "lstrip": false,
94
+ "rstrip": false,
95
+ "normalized": false,
96
+ "special": true
97
+ },
98
+ "50040": {
99
+ "content": "<|python_tag|>",
100
+ "single_word": false,
101
+ "lstrip": false,
102
+ "rstrip": false,
103
+ "normalized": false,
104
+ "special": true
105
+ },
106
+ "50041": {
107
+ "content": "<|reserved_special_token_3|>",
108
+ "single_word": false,
109
+ "lstrip": false,
110
+ "rstrip": false,
111
+ "normalized": false,
112
+ "special": true
113
+ },
114
+ "50042": {
115
+ "content": "<|reserved_special_token_4|>",
116
+ "single_word": false,
117
+ "lstrip": false,
118
+ "rstrip": false,
119
+ "normalized": false,
120
+ "special": true
121
+ },
122
+ "50043": {
123
+ "content": "<|reserved_special_token_5|>",
124
+ "single_word": false,
125
+ "lstrip": false,
126
+ "rstrip": false,
127
+ "normalized": false,
128
+ "special": true
129
+ },
130
+ "50044": {
131
+ "content": "<|reserved_special_token_6|>",
132
+ "single_word": false,
133
+ "lstrip": false,
134
+ "rstrip": false,
135
+ "normalized": false,
136
+ "special": true
137
+ },
138
+ "50045": {
139
+ "content": "<|reserved_special_token_7|>",
140
+ "single_word": false,
141
+ "lstrip": false,
142
+ "rstrip": false,
143
+ "normalized": false,
144
+ "special": true
145
+ },
146
+ "50046": {
147
+ "content": "<|reserved_special_token_8|>",
148
+ "single_word": false,
149
+ "lstrip": false,
150
+ "rstrip": false,
151
+ "normalized": false,
152
+ "special": true
153
+ },
154
+ "50047": {
155
+ "content": "<|reserved_special_token_9|>",
156
+ "single_word": false,
157
+ "lstrip": false,
158
+ "rstrip": false,
159
+ "normalized": false,
160
+ "special": true
161
+ }
162
+ }
163
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:71068a80bacd93857bb4df421abb9e81e5fd75c4b132861936dcad82abc86d73
3
+ size 5777