Add handoff2 GRPO checkpoint-500 adapter

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 sft-final/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 grpo-lora-final/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+handoff2-grpo-checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text

handoff2-grpo-checkpoint-500/README.md

@@ -0,0 +1,57 @@
+---
+library_name: peft
+pipeline_tag: text-generation
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+tags:
+- game24
+- grpo
+- lora
+- qwen2.5
+- trl
+---
+
+# Game24 RL Handoff2 GRPO Checkpoint-500
+
+This folder contains the handoff2 LoRA GRPO adapter for the Game24 RL course project.
+
+## Lineage
+
+- Base public model: `Qwen/Qwen2.5-1.5B-Instruct`
+- Project SFT base used for this adapter: `Prometheus17/game24-rl`, subfolder `sft-final`
+- This adapter: `Prometheus17/game24-rl`, subfolder `handoff2-grpo-checkpoint-500`
+
+The adapter was trained from the project handoff1 SFT-final checkpoint, not directly from the public Qwen base model.
+
+## Training Summary
+
+- Method: LoRA GRPO
+- Reward profile: `closure_control_smooth`
+- Max completion length: 4096
+- Num generations: 8
+- Learning rate: 5e-6
+- Beta: 0
+- Scale rewards: none
+- Loss type: dr_grpo
+- Delivered checkpoint: step 500
+
+## Evaluation Summary
+
+Direct greedy decoding, `max_new_tokens=4096`, strict verifier, repo-local validation/test split:
+
+| split | solved / total | accuracy |
+| --- | ---: | ---: |
+| validation | 128/136 | 94.12% |
+| test | 129/137 | 94.16% |
+| validation + test | 257/273 | 94.14% |
+
+## Loading Sketch
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+repo = "Prometheus17/game24-rl"
+base = AutoModelForCausalLM.from_pretrained(repo, subfolder="sft-final", device_map="auto")
+tokenizer = AutoTokenizer.from_pretrained(repo, subfolder="sft-final")
+model = PeftModel.from_pretrained(base, repo, subfolder="handoff2-grpo-checkpoint-500")
+```

handoff2-grpo-checkpoint-500/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "outputs/experiments/baseline_format_v2_full_5000_from800/final",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "gate_proj",
+    "v_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

handoff2-grpo-checkpoint-500/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7dff1e680ac45bdd33a0f2cf3ac7c9d2cbc13400eb0c39ece06b784b6730587c
+size 73911112

handoff2-grpo-checkpoint-500/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

handoff2-grpo-checkpoint-500/run-config.json

@@ -0,0 +1,22 @@
+{
+  "schema_version": "handoff1-grpo-final-run-config-v1",
+  "base_checkpoint": "outputs/experiments/baseline_format_v2_full_5000_from800/final",
+  "manifest": "data/processed/splits/standard-game24-v1.json",
+  "tot_manifest": "data/processed/splits/official-tot-overnight-v1.json",
+  "tot_split": "tot_all_1362",
+  "split": "train",
+  "prompt_style": "qwen_chat",
+  "max_new_tokens": 4096,
+  "num_generations": 8,
+  "max_steps": 2000,
+  "save_steps": 500,
+  "logging_steps": 1,
+  "learning_rate": "5e-6",
+  "beta": 0,
+  "scale_rewards": "none",
+  "gradient_accumulation_steps": 8,
+  "reward_profile": "closure_control_smooth",
+  "mask_truncated_completions": false,
+  "remove_unused_columns": false,
+  "training_mode": "LoRA adapter"
+}

handoff2-grpo-checkpoint-500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

handoff2-grpo-checkpoint-500/tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": null
+}

handoff2-grpo-checkpoint-500/train-run-metadata.json

@@ -0,0 +1,49 @@
+{
+  "beta": 0.0,
+  "gradient_accumulation_steps": 8,
+  "grpo_config_kwargs": {
+    "beta": 0.0,
+    "bf16": true,
+    "gradient_accumulation_steps": 8,
+    "learning_rate": 5e-06,
+    "logging_steps": 1,
+    "loss_type": "dr_grpo",
+    "mask_truncated_completions": false,
+    "max_completion_length": 4096,
+    "max_steps": 2000,
+    "num_generations": 8,
+    "output_dir": "outputs/experiments/handoff1_grpo_closure_control_smooth_v1_2000/train",
+    "per_device_train_batch_size": 1,
+    "remove_unused_columns": false,
+    "report_to": [
+      "tensorboard"
+    ],
+    "save_steps": 500,
+    "scale_rewards": "none",
+    "temperature": 0.8,
+    "top_p": 0.95
+  },
+  "initial_adapter": null,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_rank": 16,
+  "manifest": "data/processed/splits/standard-game24-v1.json",
+  "mask_truncated_completions": false,
+  "max_prompt_length_applied_by_trl": false,
+  "max_prompt_length_requested": 256,
+  "max_steps": 2000,
+  "model_name_or_path": "outputs/experiments/baseline_format_v2_full_5000_from800/final",
+  "num_generations": 8,
+  "output_dir": "outputs/experiments/handoff1_grpo_closure_control_smooth_v1_2000/train",
+  "peft_mode": "lora",
+  "pool_manifest": "outputs/experiments/handoff1_grpo_closure_control_smooth_v1_2000/pool/pool-manifest.json",
+  "prompt_records": 1462,
+  "remove_unused_columns": false,
+  "reward_profile": "closure_control_smooth",
+  "scale_rewards": "none",
+  "schema_version": "game24-grpo-train-run-v1",
+  "skipped_grpo_config_kwargs": {},
+  "split": "train",
+  "temperature": 0.8,
+  "top_p": 0.95
+}