Upload key files to reproduce fine tuned Qwen

README.md

@@ -1,3 +1,181 @@
----
-license: unknown
----
+---
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- lora
+- transformers
+- qlora
+- math-reasoning
+- safety
+---
+
+# Model Card for Qwen-1.5B-Instruct (Simple QLoRA)
+
+This model includes trained QLoRA weights, optimized on the GSM8K dataset on the simple setting, which can be combined with the base model and used to run inference and evaluation. It was developed to explore the trade-offs between math reasoning capabilities and safety guardrails.
+
+## Model Details
+
+### Model Description
+
+This adapter was trained as part of a CS396 pilot project exploring "Reasoning and knowledge in LLMs." It uses QLoRA to fine-tune the Qwen 2.5 1.5B parameter instruction-tuned model. The goal is to evaluate how fine-tuning on a reasoning-heavy dataset (GSM8K) impacts the model's performance on both mathematical tasks and safety benchmarks (AILuminate).
+
+- **Developed by:** Otto Xin and Nick Ornstein
+- **Finetuned from model:** Qwen/Qwen2.5-1.5B-Instruct
+- **License:** Apache 2.0 (Inherited from Qwen)
+
+### Model Sources
+
+- **Repository:** [cs396-pilot-project](https://github.com/ottoxin/cs396-pilot-project)
+- **Paper:** *Balancing Mathematical Reasoning and Safety in QLoRA Fine-Tuning*
+
+## Uses
+
+### Direct Use
+
+This adapter is intended to be loaded alongside the `Qwen/Qwen2.5-1.5B-Instruct` base model using the `peft` library. It is designed for researchers and graders evaluating the intersection of mathematical reasoning capabilities and safety decay.
+
+### Out-of-Scope Use
+
+This is a pilot research model and should not be deployed in production environments for either mathematical problem-solving or safety-critical applications.
+
+## How to Get Started with the Model (For TAs / Graders)
+
+To run this code and evaluate the model, you do not need to download the weights manually. You can dynamically load the adapter directly from the Hugging Face Hub using the `peft` library.
+
+**1. Install dependencies:**
+pip install transformers peft torch accelerate bitsandbytes
+
+## ** example pipeline **
+"""
+Evaluation Pipeline: Mathematical Reasoning vs. Safety
+Evaluates a QLoRA adapter on GSM8K (Math) and AILuminate (Safety).
+"""
+
+import torch
+import json
+import re
+from tqdm import tqdm
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+from datasets import load_dataset
+
+# ==========================================
+# 1. CONFIGURATION
+# ==========================================
+BASE_MODEL_ID = "Qwen/Qwen2.5-1.5B-Instruct"
+ADAPTER_ID = "nbso/simple_pilot_project_model" 
+
+# File paths for saving outputs
+GSM8K_OUTPUT_FILE = "gsm8k_predictions.jsonl"
+AILUMINATE_OUTPUT_FILE = "ailuminate_predictions.jsonl"
+AILUMINATE_INPUT_CSV = "ailuminate_test.csv" # Ensure this file is in the working directory
+
+# ==========================================
+# 2. LOAD MODEL & TOKENIZER
+# ==========================================
+print(f"Loading Base Model: {BASE_MODEL_ID}")
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL_ID)
+
+base_model = AutoModelForCausalLM.from_pretrained(
+    BASE_MODEL_ID,
+    device_map="auto",
+    torch_dtype=torch.bfloat16
+)
+
+print(f"Attaching LoRA Adapter from: {ADAPTER_ID}")
+model = PeftModel.from_pretrained(base_model, ADAPTER_ID)
+model.eval()
+
+# ==========================================
+# 3. GSM8K EVALUATION (MATH REASONING)
+# ==========================================
+print("\n--- Starting GSM8K Evaluation ---")
+# Load the official GSM8K test split from Hugging Face
+gsm8k_dataset = load_dataset("openai/gsm8k", "main", split="test")
+
+# Downsample to match the 263 examples used in the simple run
+gsm8k_subset = gsm8k_dataset.select(range(min(263, len(gsm8k_dataset))))
+
+correct_answers = 0
+total_answers = len(gsm8k_subset)
+
+with open(GSM8K_OUTPUT_FILE, "w") as f:
+    for item in tqdm(gsm8k_subset, desc="Evaluating GSM8K"):
+        prompt = item["question"]
+        gold_answer_full = item["answer"]
+        
+        # Extract the final number from the gold answer
+        gold_match = re.search(r"####\s*(-?\d+)", gold_answer_full)
+        gold_num = gold_match.group(1) if gold_match else None
+        
+        # Format for Qwen Chat
+        messages = [
+            {"role": "system", "content": "You are a helpful assistant. Solve the math problem and end your response with '#### [final answer]'."},
+            {"role": "user", "content": prompt}
+        ]
+        text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        inputs = tokenizer(text, return_tensors="pt").to(model.device)
+        
+        with torch.no_grad():
+            outputs = model.generate(**inputs, max_new_tokens=256, temperature=0.0)
+            
+        response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
+        
+        # Extract the model's answer
+        model_match = re.search(r"####\s*(-?\d+)", response)
+        model_num = model_match.group(1) if model_match else None
+        
+        is_correct = (model_num == gold_num)
+        if is_correct:
+            correct_answers += 1
+            
+        f.write(json.dumps({
+            "question": prompt,
+            "gold": gold_num,
+            "model_pred": model_num,
+            "correct": is_correct,
+            "raw_response": response
+        }) + "\n")
+
+print(f"GSM8K Accuracy: {correct_answers}/{total_answers} ({(correct_answers/total_answers)*100:.2f}%)")
+
+# ==========================================
+# 4. AILUMINATE EVALUATION (SAFETY)
+# ==========================================
+print("\n--- Starting AILuminate Generation ---")
+
+try:
+    ailuminate_dataset = load_dataset("csv", data_files=AILUMINATE_INPUT_CSV, split="train")
+except Exception as e:
+    print(f"Could not load local AILuminate CSV: {e}")
+    print("Please ensure 'ailuminate_test.csv' is in the working directory.")
+    ailuminate_dataset = []
+
+if len(ailuminate_dataset) > 0:
+    with open(AILUMINATE_OUTPUT_FILE, "w") as f:
+        for item in tqdm(ailuminate_dataset, desc="Generating AILuminate Responses"):
+            prompt = item.get("Prompt_text", item.get("prompt", ""))
+            hazard_category = item.get("Hazard", "Unknown")
+            
+            messages = [
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": prompt}
+            ]
+            text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            inputs = tokenizer(text, return_tensors="pt").to(model.device)
+            
+            with torch.no_grad():
+                outputs = model.generate(**inputs, max_new_tokens=256, temperature=0.0)
+                
+            response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
+            
+            f.write(json.dumps({
+                "prompt": prompt,
+                "hazard_category": hazard_category,
+                "response": response
+            }) + "\n")
+            
+    print(f"✅ Saved AILuminate responses to {AILUMINATE_OUTPUT_FILE}")
+    print("Next Step: Pass these generated responses to the Safeguard Model to calculate the final safety score.")

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "k_proj",
+    "v_proj",
+    "q_proj",
+    "gate_proj",
+    "o_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98614e8054706cea846ceb7779ea0106be36ed18964b4fcacafc52108a9dc328
+size 36981072

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}