Hugging Face's logo

pmangesh
/
FML_project

Safetensors
Model card Files
xet
 Community
pmangesh commited on
Commit
b54c4c8
·
verified ·
1 Parent(s): c51108b

Upload folder using huggingface_hub

Browse files
Files changed (5) hide show
  1. joint_kd_only_seed1/run_info.json +33 -0
  2. joint_kd_only_seed1/stage1_joint/config.json +61 -0
  3. joint_kd_only_seed1/stage1_joint/generation_config.json +14 -0
  4. joint_kd_only_seed1/stage1_joint/log.json +0 -0
  5. joint_kd_only_seed1/stage1_joint/model.safetensors +3 -0
joint_kd_only_seed1/run_info.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "setting": "KD only",
3
+ "alpha": 1.0,
4
+ "teacher_sft": false,
5
+ "seed": 1,
6
+ "steps": 897,
7
+ "gpu_student": "NVIDIA RTX A6000",
8
+ "gpu_teachers": "NVIDIA RTX A6000",
9
+ "load_h": 0.006147197286287944,
10
+ "train_h": 3.2132003085480796,
11
+ "val_start": {
12
+ "step": 0,
13
+ "math_kl": 0.15910778235068246,
14
+ "math_ce": 0.6515277156879916,
15
+ "code_kl": 0.14790875576132695,
16
+ "code_ce": 0.30340147782401555,
17
+ "math_hard_kl": 0.24894053959908363,
18
+ "math_hard_ce": 0.6765219455834559
19
+ },
20
+ "val_end": {
21
+ "step": 897,
22
+ "math_kl": 0.0773266607930298,
23
+ "math_ce": 0.7660258270123834,
24
+ "code_kl": 0.08573625555040962,
25
+ "code_ce": 0.30656296972393954,
26
+ "math_hard_kl": 0.08857994466653411,
27
+ "math_hard_ce": 0.746503098308834
28
+ },
29
+ "peak_gb_student": 36.57978296279907,
30
+ "peak_gb_teachers": 29.54020071029663,
31
+ "exp": 2,
32
+ "student": "Qwen/Qwen2.5-1.5B-Instruct"
33
+ }
joint_kd_only_seed1/stage1_joint/config.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": 151643,
7
+ "dtype": "bfloat16",
8
+ "eos_token_id": 151645,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 1536,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 8960,
13
+ "layer_types": [
14
+ "full_attention",
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention",
41
+ "full_attention"
42
+ ],
43
+ "max_position_embeddings": 32768,
44
+ "max_window_layers": 21,
45
+ "model_type": "qwen2",
46
+ "num_attention_heads": 12,
47
+ "num_hidden_layers": 28,
48
+ "num_key_value_heads": 2,
49
+ "pad_token_id": null,
50
+ "rms_norm_eps": 1e-06,
51
+ "rope_parameters": {
52
+ "rope_theta": 1000000.0,
53
+ "rope_type": "default"
54
+ },
55
+ "sliding_window": null,
56
+ "tie_word_embeddings": true,
57
+ "transformers_version": "5.20.2",
58
+ "use_cache": false,
59
+ "use_sliding_window": false,
60
+ "vocab_size": 151936
61
+ }
joint_kd_only_seed1/stage1_joint/generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 151645,
6
+ 151643
7
+ ],
8
+ "pad_token_id": 151643,
9
+ "repetition_penalty": 1.1,
10
+ "temperature": 0.7,
11
+ "top_k": 20,
12
+ "top_p": 0.8,
13
+ "transformers_version": "5.20.2"
14
+ }
joint_kd_only_seed1/stage1_joint/log.json ADDED
The diff for this file is too large to render. See raw diff
 
joint_kd_only_seed1/stage1_joint/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e1fdcfe6b13a697eaaafb71802281bb6cbc15d7f4616f195c520a6c6ffcbe89
3
+ size 3087467144