suyeon918 commited on
Commit
8af56bb
·
verified ·
1 Parent(s): ec6d05b

v14: 장애 신고·오류 제보 보강

Browse files
.gitattributes CHANGED
@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  v13/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
 
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  v13/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ v14/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
v14/encoder/config.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "ModernBertForMaskedLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 2,
8
+ "classifier_activation": "gelu",
9
+ "classifier_bias": false,
10
+ "classifier_dropout": 0.0,
11
+ "classifier_pooling": "mean",
12
+ "cls_token_id": 1,
13
+ "decoder_bias": true,
14
+ "deterministic_flash_attn": false,
15
+ "dtype": "float32",
16
+ "embedding_dropout": 0.0,
17
+ "eos_token_id": 1,
18
+ "global_attn_every_n_layers": 3,
19
+ "gradient_checkpointing": false,
20
+ "hidden_activation": "gelu",
21
+ "hidden_size": 768,
22
+ "initializer_cutoff_factor": 2.0,
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 1152,
25
+ "layer_norm_eps": 1e-05,
26
+ "layer_types": [
27
+ "full_attention",
28
+ "sliding_attention",
29
+ "sliding_attention",
30
+ "full_attention",
31
+ "sliding_attention",
32
+ "sliding_attention",
33
+ "full_attention",
34
+ "sliding_attention",
35
+ "sliding_attention",
36
+ "full_attention",
37
+ "sliding_attention",
38
+ "sliding_attention",
39
+ "full_attention",
40
+ "sliding_attention",
41
+ "sliding_attention",
42
+ "full_attention",
43
+ "sliding_attention",
44
+ "sliding_attention",
45
+ "full_attention",
46
+ "sliding_attention",
47
+ "sliding_attention",
48
+ "full_attention"
49
+ ],
50
+ "local_attention": 128,
51
+ "mask_token_id": 4,
52
+ "max_position_embeddings": 8192,
53
+ "mlp_bias": false,
54
+ "mlp_dropout": 0.0,
55
+ "model_type": "modernbert",
56
+ "norm_bias": false,
57
+ "norm_eps": 1e-05,
58
+ "num_attention_heads": 12,
59
+ "num_hidden_layers": 22,
60
+ "pad_token_id": 0,
61
+ "position_embedding_type": "sans_pos",
62
+ "repad_logits_with_grad": false,
63
+ "rope_parameters": {
64
+ "full_attention": {
65
+ "rope_theta": 160000,
66
+ "rope_type": "default"
67
+ },
68
+ "sliding_attention": {
69
+ "rope_theta": 160000,
70
+ "rope_type": "default"
71
+ }
72
+ },
73
+ "sep_token_id": 1,
74
+ "sparse_pred_ignore_index": -100,
75
+ "sparse_prediction": false,
76
+ "tie_word_embeddings": true,
77
+ "transformers_version": "5.22.2",
78
+ "vocab_size": 256000
79
+ }
v14/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86fadec473939249870d3d5fb0ea3dfd25d2176c0aa6facf5564c3d8145d151c
3
+ size 643835524
v14/questions.json ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "category": {
3
+ "type": "choice",
4
+ "instructions": "\uc774\uba54\uc77c\uc744 \uc5b4\ub290 \ub2f4\ub2f9 \ubd84\uc57c\uc5d0\uc11c \ucc98\ub9ac\ud574\uc57c \ud558\ub294\uac00?",
5
+ "criteria": {
6
+ "billing": "\uccad\uad6c\u00b7\uacb0\uc81c\u00b7\ud658\ubd88",
7
+ "technical": "\uae30\uc220 \ubb38\uc81c",
8
+ "sales": "\uc601\uc5c5\u00b7\uad6c\ub9e4",
9
+ "security": "\ubcf4\uc548",
10
+ "hr": "\uc778\uc0ac",
11
+ "other": "\uae30\ud0c0"
12
+ }
13
+ },
14
+ "is_spam": {
15
+ "type": "noul",
16
+ "instructions": "\uc6d0\ud558\uc9c0 \uc54a\ub294 \uc2a4\ud338 \ub610\ub294 \ub300\ub7c9 \ud64d\ubcf4 \uba54\uc77c\uc778\uac00?",
17
+ "criteria": {
18
+ "true": "\ucc38: \uc2a4\ud338\u00b7\ub300\ub7c9 \ud64d\ubcf4",
19
+ "false": "\uac70\uc9d3: \uc815\uc0c1 \uc774\uba54\uc77c"
20
+ }
21
+ },
22
+ "is_phishing": {
23
+ "type": "noul",
24
+ "instructions": "\uae08\uc804\u00b7\uc778\uc99d\uc815\ubcf4\u00b7\uac1c\uc778\uc815\ubcf4\ub97c \ud0c8\ucde8\ud558\ub824\ub294 \ud53c\uc2f1\uc774\ub098 \uc0ac\uae30\uc778\uac00?",
25
+ "criteria": {
26
+ "true": "\ucc38: \ud53c\uc2f1\u00b7\uc0ac\uae30\u00b7\ubd80\uc815\ud589\uc704",
27
+ "false": "\uac70\uc9d3: \uc815\uc0c1 \uc774\uba54\uc77c"
28
+ }
29
+ },
30
+ "urgency": {
31
+ "type": "score",
32
+ "instructions": "\ubcf8\ubb38 \uc694\uccad\uc744 \uc5bc\ub9c8\ub098 \uae34\uae09\ud558\uac8c \ucc98\ub9ac\ud574\uc57c \ud558\ub294\uac00?",
33
+ "criteria": [
34
+ "\ub0ae\uc74c: \uc2dc\uac04 \uc555\ubc15 \uc5c6\uc74c",
35
+ "\ubcf4\ud1b5: \ube60\ub978 \ud655\uc778 \ud544\uc694",
36
+ "\uae34\uae09: \uc5c5\ubb34 \ucc28\ub2e8 \ub610\ub294 \ud655\uc815 \uae30\ud55c"
37
+ ]
38
+ },
39
+ "needs_reply": {
40
+ "type": "noul",
41
+ "instructions": "\ubc1c\uc2e0\uc790\uac00 \ub2f5\uc7a5\uc744 \uae30\ub300\ud558\ub294\uac00?",
42
+ "criteria": {
43
+ "true": "\ucc38: \ub2f5\uc7a5\u00b7\ucc98\ub9ac\ub97c \uae30\ub300\ud568",
44
+ "false": "\uac70\uc9d3: \uc54c\ub9bc\u00b7\uc548\ub0b4\ub77c \ub2f5\uc7a5 \ubd88\ud544\uc694"
45
+ }
46
+ },
47
+ "action": {
48
+ "type": "choice",
49
+ "instructions": "\uc774 \uba54\uc77c\uc744 \ubc1b\uc740 \ub300\ud45c\uac00 \ub2e4\uc74c\uc5d0 \ud574\uc57c \ud560 \uc77c\uc740 \ubb34\uc5c7\uc778\uac00?",
50
+ "criteria": {
51
+ "\ud68c\uc2e0": "\ub2f5\uc7a5 \uba54\uc77c \ubcf8\ubb38\ub9cc\uc73c\ub85c \ub05d\ub0a8 (\uc9c8\ubb38 \ub2f5\ubcc0, \ud655\uc778, \uc218\ub77d\u00b7\uac70\uc808 \ud1b5\ubcf4)",
52
+ "\ubb38\uc11c\uc791\uc131": "\uc0c1\ub300\uac00 \uacac\uc801\u00b7\uc790\ub8cc\u00b7\uacc4\uc57d\uc11c\ub97c '\uc694\uccad'\ud574\uc11c \ub300\ud45c\uac00 \ubb38\uc11c\ub97c \ub9cc\ub4e4\uc5b4 \ubcf4\ub0b4\uc57c \ud568",
53
+ "\uc77c\uc815\ub4f1\ub85d": "\ubbf8\ud305\u00b7\ub0a9\uae30\u00b7\ub9c8\uac10 \uc77c\uc815\uc744 \ub4f1\ub85d\ud558\ub294 \uac83\uc774 \ud575\uc2ec",
54
+ "\uacb0\uc7ac\uc694\uccad": "\uc0c1\ub300\uac00 \uacac\uc801\u00b7\uccad\uad6c\uc11c\u00b7\uacc4\uc57d\uc11c\ub97c '\ubcf4\ub0b4\uc640\uc11c' \ub300\ud45c\uac00 \uc218\ub77d\u00b7\uc9c0\uae09\u00b7\ud658\ubd88\uc744 \uacb0\uc815\ud574\uc57c \ud568",
55
+ "\ucc98\ub9ac\ubd88\ud544\uc694": "\ub2e8\uc21c \uc548\ub0b4\u00b7\uad11\uace0\u00b7\uc774\ubbf8 \ud574\uacb0\ub41c \uac74"
56
+ }
57
+ },
58
+ "complexity": {
59
+ "type": "choice",
60
+ "instructions": "\ub300\ud45c\uac00 \ub9cc\ub4e4\uc5b4\uc57c \ud560 \uacb0\uacfc\ubb3c\uc774 \uba87 \uac1c\uc778\uac00? (\uc608\uc0b0\u00b7\ubc30\uc1a1\u00b7\uacb0\uc81c \uc870\uac74\uc774 \uc5ec\ub7ec \uac1c \uc801\ud600 \uc788\uc5b4\ub3c4 \uacac\uc801\uc11c \ud55c \uc7a5\uc774\uba74 \uacb0\uacfc\ubb3c\uc740 \ud558\ub098)",
61
+ "criteria": {
62
+ "\ub2e8\uc21c": "\uacb0\uacfc\ubb3c\uc774 \ud558\ub098\ub2e4 (\ud68c\uc2e0 1\ud1b5, \ub610\ub294 \ubb38\uc11c 1\uac1c)",
63
+ "\ubcf5\ud569": "\uc11c\ub85c \ub2e4\ub978 \uacb0\uacfc\ubb3c\uc774 2\uac1c \uc774\uc0c1\uc774\ub2e4 (\uc608: \uacac\uc801\uc11c+\uacc4\uc57d\uc11c+\uc77c\uc815), \ub610\ub294 \uc5ec\ub7ec \ubb38\uc11c\ub97c \uc885\ud569\ud574\uc57c \ud55c\ub2e4"
64
+ }
65
+ },
66
+ "sensitive": {
67
+ "type": "noul",
68
+ "instructions": "\uc774 \uba54\uc77c\uc5d0 \ud2b9\uc815 \uc0ac\ub78c\uc758 \uc778\uc801\uc0ac\ud56d(\uc8fc\ubbfc\ub4f1\ub85d\ubc88\ud638, \uc9d1 \uc8fc\uc18c, \uc0dd\ub144\uc6d4\uc77c, \uac1c\uc778 \uacc4\uc88c, \uac74\uac15 \uc815\ubcf4, \uc778\uc0ac \ud3c9\uac00, \uacc4\uc815 \ubcf4\uc548 \uc815\ubcf4\uc778 \ub85c\uadf8\uc778 IP\u00b7\uae30\uae30\u00b7\uc704\uce58)\uc774\ub098 \ud68c\uc0ac \ub0b4\ubd80 \uae30\ubc00(\uc6d0\uac00, \ub9c8\uc9c4, \uacf5\uac1c \uae08\uc9c0 \ub2e8\uac00, \uc7ac\ubb34 \uc0ac\uc815)\uc774 \uc2e4\uc81c\ub85c \uc801\ud600 \uc788\ub294\uac00?",
69
+ "criteria": {
70
+ "false": "\uc77c\ubc18 \uc5c5\ubb34 \ub0b4\uc6a9\ubfd0\uc774\ub2e4(\uacac\uc801 \uc694\uccad, \uc77c\uc815 \ubb38\uc758, \uc548\ub0b4, \uacf5\uc9c0). \uc778\uc801\uc0ac\ud56d\uc774\ub098 \uae30\ubc00\uc774 \uc2e4\uc81c\ub85c \uc801\ud600 \uc788\uc9c0 \uc54a\ub2e4.",
71
+ "true": "\ud2b9\uc815 \uc0ac\ub78c\uc758 \uc778\uc801\uc0ac\ud56d\uc774\ub098 \ud68c\uc0ac \ub0b4\ubd80 \uae30\ubc00\uc774 \uc2e4\uc81c\ub85c \uc801\ud600 \uc788\ub2e4."
72
+ }
73
+ }
74
+ }
v14/rl_agent_config.json ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "encoder": "jhu-clsp/mmBERT-base",
3
+ "head_layers": 2,
4
+ "max_len": 2048,
5
+ "head_max_len": 256,
6
+ "max_prefixes": 6,
7
+ "act_costs": {
8
+ "escalate": 0.5
9
+ },
10
+ "cost_wrong_act": 3.0,
11
+ "amp_dtype": "bf16",
12
+ "model_name": "rl-agent",
13
+ "temperature": [
14
+ 3.7832961082458496,
15
+ 1.0,
16
+ 3.33156156539917
17
+ ],
18
+ "training": {
19
+ "updates": 15987,
20
+ "epochs_completed": 4,
21
+ "hours": 4.97,
22
+ "world_size": 1,
23
+ "fine_tuned_from_checkpoint": false,
24
+ "laya_train": {
25
+ "epochs": 5,
26
+ "micro_batch": 2,
27
+ "grad_accum": 8,
28
+ "encoder_lr": 2.5e-05,
29
+ "head_lr": 0.0001,
30
+ "min_lr": 1e-06,
31
+ "weight_decay": 0.01,
32
+ "grad_clip": 1.0,
33
+ "loss": "rlcd",
34
+ "rl_samples": 4,
35
+ "sigma_start": 0.4,
36
+ "sigma_end": 0.1,
37
+ "w_sph": 0.75,
38
+ "w_rps": 1.0,
39
+ "shuffle_options": [
40
+ "choice"
41
+ ],
42
+ "option_layout": null,
43
+ "freeze_encoder": false,
44
+ "calib_max": 400,
45
+ "calib_frac": 0.1,
46
+ "calib_seed": 20260922,
47
+ "seed": 0,
48
+ "max_len": 2048,
49
+ "head_max_len": null,
50
+ "amp": null,
51
+ "gradient_checkpointing": null,
52
+ "log_every": 100,
53
+ "label_smoothing": 0.0,
54
+ "text_column": "text",
55
+ "label_column": "label",
56
+ "question_id": "label",
57
+ "instructions": null,
58
+ "eval_data": "data\\sets\\val.jsonl",
59
+ "target_error": 0.1,
60
+ "min_abstain_n": 10
61
+ },
62
+ "laya_train_calibration": {
63
+ "choice": {
64
+ "items": 139,
65
+ "temperature": 3.7832961082458496,
66
+ "issues": [],
67
+ "accuracy": 0.9712230215827338,
68
+ "mean_confidence": 0.9759824520505122
69
+ },
70
+ "score": {
71
+ "items": 44,
72
+ "temperature": 1.0,
73
+ "issues": [
74
+ "fitted on only 44 calibration items",
75
+ "the fit came back unchanged at 1.0 (accuracy 1.00, mean confidence 1.000 on the calibration items), which usually means they gave it nothing to correct; this type's confidences are uncalibrated"
76
+ ],
77
+ "accuracy": 1.0,
78
+ "mean_confidence": 0.999999929841763
79
+ },
80
+ "noul": {
81
+ "items": 217,
82
+ "temperature": 3.33156156539917,
83
+ "issues": [],
84
+ "accuracy": 0.9953917050691244,
85
+ "mean_confidence": 0.9961188719812915
86
+ }
87
+ },
88
+ "train_report": {
89
+ "eval_source": "data\\sets\\val.jsonl",
90
+ "eval_mode": "overlapping_eval",
91
+ "is_held_out": false,
92
+ "eval_items": 3118,
93
+ "note": "Evaluation set contains rows that overlap training data (8/3118 items); metrics do not reflect strictly independent held-out evaluation.",
94
+ "before": {
95
+ "items": 3118,
96
+ "loss": 2.4687,
97
+ "accuracy": 0.4641,
98
+ "mean_confidence": 0.8268,
99
+ "ece": 0.3628,
100
+ "brier": 0.8545,
101
+ "brier_top1": 0.3896,
102
+ "by_type": {
103
+ "choice": {
104
+ "items": 1164,
105
+ "loss": 1.7198,
106
+ "accuracy": 0.5112,
107
+ "mean_confidence": 0.7506,
108
+ "ece": 0.2394,
109
+ "brier": 0.7207,
110
+ "brier_top1": 0.2891
111
+ },
112
+ "score": {
113
+ "items": 374,
114
+ "loss": 2.0527,
115
+ "accuracy": 0.2941,
116
+ "mean_confidence": 0.7141,
117
+ "ece": 0.42,
118
+ "brier": 0.9597,
119
+ "brier_top1": 0.3878
120
+ },
121
+ "noul": {
122
+ "items": 1580,
123
+ "loss": 3.1189,
124
+ "accuracy": 0.4696,
125
+ "mean_confidence": 0.9097,
126
+ "ece": 0.4401,
127
+ "brier": 0.9281,
128
+ "brier_top1": 0.464
129
+ }
130
+ }
131
+ },
132
+ "after": {
133
+ "items": 3118,
134
+ "loss": 0.2564,
135
+ "accuracy": 0.9622,
136
+ "mean_confidence": 0.9868,
137
+ "ece": 0.0263,
138
+ "brier": 0.0724,
139
+ "brier_top1": 0.0355,
140
+ "by_type": {
141
+ "choice": {
142
+ "items": 1164,
143
+ "loss": 0.2698,
144
+ "accuracy": 0.9424,
145
+ "mean_confidence": 0.972,
146
+ "ece": 0.0309,
147
+ "brier": 0.1056,
148
+ "brier_top1": 0.051
149
+ },
150
+ "score": {
151
+ "items": 374,
152
+ "loss": 0.8551,
153
+ "accuracy": 0.9465,
154
+ "mean_confidence": 0.9953,
155
+ "ece": 0.0581,
156
+ "brier": 0.1106,
157
+ "brier_top1": 0.0553
158
+ },
159
+ "noul": {
160
+ "items": 1580,
161
+ "loss": 0.1047,
162
+ "accuracy": 0.9804,
163
+ "mean_confidence": 0.9958,
164
+ "ece": 0.016,
165
+ "brier": 0.0388,
166
+ "brier_top1": 0.0194
167
+ }
168
+ }
169
+ },
170
+ "comparison": {
171
+ "delta_accuracy": 0.4981,
172
+ "delta_loss": -2.2123,
173
+ "delta_ece": -0.3365,
174
+ "delta_brier": -0.7821,
175
+ "delta_brier_top1": -0.3541,
176
+ "delta_mean_confidence": 0.16
177
+ },
178
+ "calibration": {
179
+ "choice": {
180
+ "items": 139,
181
+ "temperature": 3.7832961082458496,
182
+ "issues": [],
183
+ "accuracy": 0.9712230215827338,
184
+ "mean_confidence": 0.9759824520505122
185
+ },
186
+ "score": {
187
+ "items": 44,
188
+ "temperature": 1.0,
189
+ "issues": [
190
+ "fitted on only 44 calibration items",
191
+ "the fit came back unchanged at 1.0 (accuracy 1.00, mean confidence 1.000 on the calibration items), which usually means they gave it nothing to correct; this type's confidences are uncalibrated"
192
+ ],
193
+ "accuracy": 1.0,
194
+ "mean_confidence": 0.999999929841763
195
+ },
196
+ "noul": {
197
+ "items": 217,
198
+ "temperature": 3.33156156539917,
199
+ "issues": [],
200
+ "accuracy": 0.9953917050691244,
201
+ "mean_confidence": 0.9961188719812915
202
+ }
203
+ },
204
+ "abstention_thresholds": {
205
+ "score:3-5": 1.0,
206
+ "noul:2": 0.9763,
207
+ "choice:3-5": 0.9681,
208
+ "choice:2": 0.9901,
209
+ "choice:6-10": 0.8453
210
+ },
211
+ "training": {
212
+ "epochs": 5,
213
+ "micro_batch": 2,
214
+ "grad_accum": 8,
215
+ "encoder_lr": 2.5e-05,
216
+ "head_lr": 0.0001,
217
+ "min_lr": 1e-06,
218
+ "weight_decay": 0.01,
219
+ "grad_clip": 1.0,
220
+ "loss": "rlcd",
221
+ "rl_samples": 4,
222
+ "sigma_start": 0.4,
223
+ "sigma_end": 0.1,
224
+ "w_sph": 0.75,
225
+ "w_rps": 1.0,
226
+ "shuffle_options": [
227
+ "choice"
228
+ ],
229
+ "option_layout": null,
230
+ "freeze_encoder": false,
231
+ "calib_max": 400,
232
+ "calib_frac": 0.1,
233
+ "calib_seed": 20260922,
234
+ "seed": 0,
235
+ "max_len": 2048,
236
+ "head_max_len": null,
237
+ "amp": null,
238
+ "gradient_checkpointing": null,
239
+ "log_every": 100,
240
+ "label_smoothing": 0.0,
241
+ "text_column": "text",
242
+ "label_column": "label",
243
+ "question_id": "label",
244
+ "instructions": null,
245
+ "eval_data": "data\\sets\\val.jsonl",
246
+ "target_error": 0.1,
247
+ "min_abstain_n": 10
248
+ }
249
+ },
250
+ "abstention_thresholds": {
251
+ "score:3-5": 1.0,
252
+ "noul:2": 0.9763,
253
+ "choice:3-5": 0.9681,
254
+ "choice:2": 0.9901,
255
+ "choice:6-10": 0.8453
256
+ }
257
+ },
258
+ "fine_tuned": true
259
+ }
v14/thresholds.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "thresholds": {
3
+ "conf_min": {
4
+ "category": 0.3,
5
+ "action": 0.3,
6
+ "complexity": 0.3
7
+ },
8
+ "sens_lo": 0.05,
9
+ "sens_hi": 0.5,
10
+ "cloud_max": 0.01
11
+ },
12
+ "근거": {
13
+ "category": {
14
+ "자동처리 비율": 1.0,
15
+ "자동처리 정확도": 0.942,
16
+ "목표 달성": true
17
+ },
18
+ "action": {
19
+ "자동처리 비율": 1.0,
20
+ "자동처리 정확도": 0.914,
21
+ "목표 달성": true
22
+ },
23
+ "complexity": {
24
+ "자동처리 비율": 1.0,
25
+ "자동처리 정확도": 0.973,
26
+ "목표 달성": true
27
+ },
28
+ "sens_lo 탐색값(미사용)": 0.5,
29
+ "sensitive": {
30
+ "val 민감 건수": 87,
31
+ "sens_lo": 0.05,
32
+ "Laya 단독 재현율": 0.931
33
+ }
34
+ },
35
+ "목표 정확도": 0.9,
36
+ "보정 실행": "20261118-190229_v14_val",
37
+ "val 건수": 416,
38
+ "주의": "val 416건 기준이라 오차가 큼. test로 다시 확인할 것."
39
+ }
v14/tokenizer/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbba919c6e5e492bc22beb80abb64e6fcc7e8744dc696e8388b8c977767afa4b
3
+ size 34363285
v14/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<bos>",
3
+ "clean_up_tokenization_spaces": false,
4
+ "cls_token": "<bos>",
5
+ "eos_token": "<eos>",
6
+ "extra_0": "<start_of_turn>",
7
+ "extra_1": "<end_of_turn>",
8
+ "local_files_only": false,
9
+ "mask_token": "<mask>",
10
+ "model_input_names": [
11
+ "input_ids",
12
+ "attention_mask"
13
+ ],
14
+ "model_max_length": 8192,
15
+ "model_specific_special_tokens": {
16
+ "extra_0": "<start_of_turn>",
17
+ "extra_1": "<end_of_turn>"
18
+ },
19
+ "pad_token": "<pad>",
20
+ "padding_side": "right",
21
+ "sep_token": "<eos>",
22
+ "spaces_between_special_tokens": false,
23
+ "tokenizer_class": "PreTrainedTokenizerFast",
24
+ "unk_token": "<unk>"
25
+ }
v14/train_report.json ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eval_source": "data\\sets\\val.jsonl",
3
+ "eval_mode": "overlapping_eval",
4
+ "is_held_out": false,
5
+ "eval_items": 3118,
6
+ "note": "Evaluation set contains rows that overlap training data (8/3118 items); metrics do not reflect strictly independent held-out evaluation.",
7
+ "before": {
8
+ "items": 3118,
9
+ "loss": 2.4687,
10
+ "accuracy": 0.4641,
11
+ "mean_confidence": 0.8268,
12
+ "ece": 0.3628,
13
+ "brier": 0.8545,
14
+ "brier_top1": 0.3896,
15
+ "by_type": {
16
+ "choice": {
17
+ "items": 1164,
18
+ "loss": 1.7198,
19
+ "accuracy": 0.5112,
20
+ "mean_confidence": 0.7506,
21
+ "ece": 0.2394,
22
+ "brier": 0.7207,
23
+ "brier_top1": 0.2891
24
+ },
25
+ "score": {
26
+ "items": 374,
27
+ "loss": 2.0527,
28
+ "accuracy": 0.2941,
29
+ "mean_confidence": 0.7141,
30
+ "ece": 0.42,
31
+ "brier": 0.9597,
32
+ "brier_top1": 0.3878
33
+ },
34
+ "noul": {
35
+ "items": 1580,
36
+ "loss": 3.1189,
37
+ "accuracy": 0.4696,
38
+ "mean_confidence": 0.9097,
39
+ "ece": 0.4401,
40
+ "brier": 0.9281,
41
+ "brier_top1": 0.464
42
+ }
43
+ }
44
+ },
45
+ "after": {
46
+ "items": 3118,
47
+ "loss": 0.2564,
48
+ "accuracy": 0.9622,
49
+ "mean_confidence": 0.9868,
50
+ "ece": 0.0263,
51
+ "brier": 0.0724,
52
+ "brier_top1": 0.0355,
53
+ "by_type": {
54
+ "choice": {
55
+ "items": 1164,
56
+ "loss": 0.2698,
57
+ "accuracy": 0.9424,
58
+ "mean_confidence": 0.972,
59
+ "ece": 0.0309,
60
+ "brier": 0.1056,
61
+ "brier_top1": 0.051
62
+ },
63
+ "score": {
64
+ "items": 374,
65
+ "loss": 0.8551,
66
+ "accuracy": 0.9465,
67
+ "mean_confidence": 0.9953,
68
+ "ece": 0.0581,
69
+ "brier": 0.1106,
70
+ "brier_top1": 0.0553
71
+ },
72
+ "noul": {
73
+ "items": 1580,
74
+ "loss": 0.1047,
75
+ "accuracy": 0.9804,
76
+ "mean_confidence": 0.9958,
77
+ "ece": 0.016,
78
+ "brier": 0.0388,
79
+ "brier_top1": 0.0194
80
+ }
81
+ }
82
+ },
83
+ "comparison": {
84
+ "delta_accuracy": 0.4981,
85
+ "delta_loss": -2.2123,
86
+ "delta_ece": -0.3365,
87
+ "delta_brier": -0.7821,
88
+ "delta_brier_top1": -0.3541,
89
+ "delta_mean_confidence": 0.16
90
+ },
91
+ "calibration": {
92
+ "choice": {
93
+ "items": 139,
94
+ "temperature": 3.7832961082458496,
95
+ "issues": [],
96
+ "accuracy": 0.9712230215827338,
97
+ "mean_confidence": 0.9759824520505122
98
+ },
99
+ "score": {
100
+ "items": 44,
101
+ "temperature": 1.0,
102
+ "issues": [
103
+ "fitted on only 44 calibration items",
104
+ "the fit came back unchanged at 1.0 (accuracy 1.00, mean confidence 1.000 on the calibration items), which usually means they gave it nothing to correct; this type's confidences are uncalibrated"
105
+ ],
106
+ "accuracy": 1.0,
107
+ "mean_confidence": 0.999999929841763
108
+ },
109
+ "noul": {
110
+ "items": 217,
111
+ "temperature": 3.33156156539917,
112
+ "issues": [],
113
+ "accuracy": 0.9953917050691244,
114
+ "mean_confidence": 0.9961188719812915
115
+ }
116
+ },
117
+ "abstention_thresholds": {
118
+ "score:3-5": 1.0,
119
+ "noul:2": 0.9763,
120
+ "choice:3-5": 0.9681,
121
+ "choice:2": 0.9901,
122
+ "choice:6-10": 0.8453
123
+ },
124
+ "training": {
125
+ "epochs": 5,
126
+ "micro_batch": 2,
127
+ "grad_accum": 8,
128
+ "encoder_lr": 2.5e-05,
129
+ "head_lr": 0.0001,
130
+ "min_lr": 1e-06,
131
+ "weight_decay": 0.01,
132
+ "grad_clip": 1.0,
133
+ "loss": "rlcd",
134
+ "rl_samples": 4,
135
+ "sigma_start": 0.4,
136
+ "sigma_end": 0.1,
137
+ "w_sph": 0.75,
138
+ "w_rps": 1.0,
139
+ "shuffle_options": [
140
+ "choice"
141
+ ],
142
+ "option_layout": null,
143
+ "freeze_encoder": false,
144
+ "calib_max": 400,
145
+ "calib_frac": 0.1,
146
+ "calib_seed": 20260922,
147
+ "seed": 0,
148
+ "max_len": 2048,
149
+ "head_max_len": null,
150
+ "amp": null,
151
+ "gradient_checkpointing": null,
152
+ "log_every": 100,
153
+ "label_smoothing": 0.0,
154
+ "text_column": "text",
155
+ "label_column": "label",
156
+ "question_id": "label",
157
+ "instructions": null,
158
+ "eval_data": "data\\sets\\val.jsonl",
159
+ "target_error": 0.1,
160
+ "min_abstain_n": 10
161
+ }
162
+ }