Publish v2 training results

README.md

@@ -18,7 +18,7 @@ tags:
 
 이 모델은 Countdown 문제의 결정적 보상을 사용하여 Qwen3-4B를 GRPO로 학습한 병합 모델입니다.
 숫자 4개를 각각 한 번 사용하고, 사칙연산으로 목표값을 만드는 수식을 생성합니다.
-학습 데이터는 `NotoriousH2/countdown-rlvr` revision `de9ee3df68c2606ff8000a6714366f9e2d9d1a10`를 사용했습니다.
+학습 데이터는 `NotoriousH2/countdown-rlvr`를 사용했습니다.
 
 ## 1. 사용
 
@@ -32,6 +32,10 @@ model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto")
 
 LoRA 파일과 토크나이저는 `adapter/`에 있습니다.
 학습 설정은 `training_config.json`, 전체 학습 기록은 `history.json`에서 확인할 수 있습니다.
+구간별 completion 길이와 체크포인트 재개 지점은 `training_provenance.json`에 있습니다.
+
+검증 분할의 후보별 지표와 공개 체크포인트 선택 결과는 `checkpoint_selection.json`에 있습니다.
+
 
 ## 2. 평가
 
@@ -39,8 +43,8 @@ Base와 RLVR 모델을 같은 test 분할과 생성 설정으로 평가했습니
 
 | 모델 | pass@1 | pass@8 | 유효 수식 비율 | 평균 응답 토큰 |
 |---|---:|---:|---:|---:|
-| Base | 7.8% | 29.7% | 6.3% | 974.4 |
-| RLVR | 14.1% | 54.7% | 14.6% | 967.4 |
+| Base | 24.2% | 70.3% | 23.9% | 1676.2 |
+| RLVR | 50.8% | 91.4% | 51.8% | 1700.2 |
 
 문제별 응답과 점수는 `evaluation.json`에 있습니다.
 

adapter/adapter_config.json

@@ -31,12 +31,12 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "q_proj",
     "gate_proj",
-    "v_proj",
+    "up_proj",
     "o_proj",
     "down_proj",
+    "q_proj",
+    "v_proj",
     "k_proj"
   ],
   "target_parameters": null,

adapter/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0cdbf678fc24096066564f2bd6bba0b4b15ba78bc97e1fcfef69a1731d75136d
+oid sha256:9335913eef55f0b769962c8e9ca96d1df1d98aeab3126aaed3bf773a7b8c9c99
 size 132187888

adapter/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c351294dc1999a8e4eb150bd5c139f42a254c1ebf4cf7ac4be7dadb4d94d55ef
+oid sha256:db2fa40b501c6ae645899142e105591000dfbd7a6c88f33caa64e3be715dec4c
 size 7697

checkpoint_selection.json

@@ -0,0 +1,46 @@
+{
+  "schema_version": 1,
+  "validation_path": "data/countdown/validation.jsonl",
+  "validation_samples": 64,
+  "completions_per_problem": 8,
+  "decoding": {
+    "max_tokens": 3072,
+    "temperature": 0.6,
+    "top_p": 0.95,
+    "top_k": 20,
+    "seed": 42
+  },
+  "candidates": [
+    {
+      "label": "checkpoint-360",
+      "pass_at_1": 0.5625,
+      "pass_at_8": 0.875,
+      "valid_expression_rate": 0.5234375,
+      "truncated_completion_rate": 0.166015625,
+      "mean_response_tokens": 1584.228515625
+    },
+    {
+      "label": "step-480",
+      "pass_at_1": 0.453125,
+      "pass_at_8": 0.953125,
+      "valid_expression_rate": 0.529296875,
+      "truncated_completion_rate": 0.185546875,
+      "mean_response_tokens": 1569.1875
+    }
+  ],
+  "selection_criterion": [
+    "pass_at_1",
+    "pass_at_8",
+    "lower_truncated_completion_rate"
+  ],
+  "selected": {
+    "label": "checkpoint-360",
+    "step": 360,
+    "canonical_adapter": "adapter",
+    "canonical_merged_model": "merged"
+  },
+  "backup": {
+    "label": "step-480",
+    "local_path": "candidates/step-480"
+  }
+}

config.json

@@ -4,7 +4,7 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": null,
+  "bos_token_id": 151643,
   "dtype": "bfloat16",
   "eos_token_id": 151645,
   "head_dim": 128,
@@ -56,7 +56,7 @@
   "num_attention_heads": 32,
   "num_hidden_layers": 36,
   "num_key_value_heads": 8,
-  "pad_token_id": 151643,
+  "pad_token_id": null,
   "rms_norm_eps": 1e-06,
   "rope_parameters": {
     "rope_theta": 1000000,
@@ -65,7 +65,7 @@
   "sliding_window": null,
   "tie_word_embeddings": true,
   "transformers_version": "5.14.1",
-  "use_cache": false,
+  "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936
 }

evaluation.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fdd11b2ff8ece98402d171cc0bf1a165c06b4e1062600fa1e90a3b79fbae4320
-size 14030557
+oid sha256:476843ae50f7acfa87159ff8507ca6d608b333165d1cc494afbca75e18d17884
+size 21912685

generation_config.json

@@ -1,4 +1,5 @@
 {
+  "bos_token_id": 151643,
   "do_sample": true,
   "eos_token_id": [
     151645,

metrics.json

@@ -3,19 +3,19 @@
   "models": [
     {
       "model": "Base",
-      "pass@1": 0.078125,
-      "pass@8": 0.296875,
-      "valid_expression_rate": 0.06298828125,
-      "valid_expression_diversity": 0.37109375,
-      "mean_response_tokens": 974.3505859375
+      "pass@1": 0.2421875,
+      "pass@8": 0.703125,
+      "valid_expression_rate": 0.23876953125,
+      "valid_expression_diversity": 1.0859375,
+      "mean_response_tokens": 1676.1796875
     },
     {
       "model": "RLVR",
-      "pass@1": 0.140625,
-      "pass@8": 0.546875,
-      "valid_expression_rate": 0.146484375,
-      "valid_expression_diversity": 0.765625,
-      "mean_response_tokens": 967.43408203125
+      "pass@1": 0.5078125,
+      "pass@8": 0.9140625,
+      "valid_expression_rate": 0.517578125,
+      "valid_expression_diversity": 1.87890625,
+      "mean_response_tokens": 1700.2041015625
     }
   ]
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7601affb485bc6a58102de35556bfcb9d74d20355026639b592a2c3490f9808f
+oid sha256:d0dfb777d6c991e2fae8fd86cc6ee31dc9fe3e2deb4b80d43911d753cb0a0627
 size 8044982080

tokenizer_config.json

@@ -20,8 +20,8 @@
     "<|image_pad|>",
     "<|video_pad|>"
   ],
-  "is_local": false,
-  "local_files_only": false,
+  "is_local": true,
+  "local_files_only": true,
   "model_max_length": 131072,
   "pad_token": "<|endoftext|>",
   "split_special_tokens": false,

training_config.json

@@ -1,9 +1,9 @@
 {
-  "output_dir": "artifacts/countdown_rlvr/checkpoints",
+  "output_dir": "artifacts/countdown_rlvr_v2/checkpoints",
   "per_device_train_batch_size": 2,
   "num_train_epochs": 1,
-  "max_steps": 120,
-  "learning_rate": 1e-05,
+  "max_steps": 480,
+  "learning_rate": 5e-06,
   "lr_scheduler_type": "cosine",
   "lr_scheduler_kwargs": null,
   "warmup_steps": 0.05,
@@ -63,10 +63,10 @@
   "include_for_metrics": [],
   "batch_eval_metrics": false,
   "save_only_model": false,
-  "save_strategy": "no",
-  "save_steps": 500,
+  "save_strategy": "steps",
+  "save_steps": 120,
   "save_on_each_node": false,
-  "save_total_limit": null,
+  "save_total_limit": 4,
   "enable_jit_checkpoint": false,
   "push_to_hub": false,
   "hub_token": "<HUB_TOKEN>",
@@ -129,15 +129,15 @@
   "cast_lm_head_to_fp32": false,
   "num_generations": 8,
   "num_generations_eval": null,
-  "max_completion_length": 1024,
+  "max_completion_length": 3072,
   "ds3_gather_for_generation": true,
   "shuffle_dataset": true,
   "pad_to_multiple_of": null,
   "generation_batch_size": 8,
   "steps_per_generation": 4,
-  "temperature": 1.0,
-  "top_p": 1.0,
-  "top_k": 0,
+  "temperature": 0.6,
+  "top_p": 0.95,
+  "top_k": 20,
   "min_p": null,
   "generation_kwargs": null,
   "chat_template_kwargs": {
@@ -158,7 +158,7 @@
   "vllm_gpu_memory_utilization": 0.3,
   "vllm_max_model_length": null,
   "vllm_tensor_parallel_size": 1,
-  "beta": 0.04,
+  "beta": 0.001,
   "num_iterations": 1,
   "epsilon": 0.2,
   "delta": null,
@@ -171,14 +171,14 @@
   "vespo_lambda_neg": 2.0,
   "importance_sampling_level": "token",
   "reward_weights": [
-    0.2,
     0.3,
+    0.2,
     0.5
   ],
   "multi_objective_aggregation": "sum_then_normalize",
   "scale_rewards": "group",
   "loss_type": "grpo",
-  "mask_truncated_completions": false,
+  "mask_truncated_completions": true,
   "sync_ref_model": false,
   "ref_model_mixup_alpha": 0.6,
   "ref_model_sync_steps": 512,

training_provenance.json

@@ -0,0 +1,28 @@
+{
+  "schema_version": 1,
+  "task": "countdown_rlvr",
+  "phases": [
+    {
+      "phase": "초기 구간",
+      "start_step": 1,
+      "end_step": 120,
+      "max_completion_length": 2048
+    },
+    {
+      "phase": "재개 구간",
+      "start_step": 121,
+      "end_step": 480,
+      "first_logged_step": 125,
+      "max_completion_length": 3072,
+      "resume_from_checkpoint": "checkpoints/checkpoint-120"
+    }
+  ],
+  "logging_steps": 5,
+  "selected_checkpoint": {
+    "step": 360,
+    "criterion": "validation pass@1",
+    "canonical_adapter": "adapter",
+    "canonical_merged_model": "merged",
+    "step_480_backup": "candidates/step-480"
+  }
+}