{ "cells": [ { "cell_type": "markdown", "id": "444cce3d", "metadata": {}, "source": [ "## To find the best hyperparameters we use optuna and wandb." ] }, { "cell_type": "code", "execution_count": 1, "id": "a380209d-f30f-4d6d-b50c-f41d877e6c8a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting optuna\n", " Downloading optuna-4.6.0-py3-none-any.whl.metadata (17 kB)\n", "Collecting alembic>=1.5.0 (from optuna)\n", " Downloading alembic-1.17.1-py3-none-any.whl.metadata (7.2 kB)\n", "Collecting colorlog (from optuna)\n", " Downloading colorlog-6.10.1-py3-none-any.whl.metadata (11 kB)\n", "Requirement already satisfied: numpy in /usr/local/lib/python3.12/dist-packages (from optuna) (2.3.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.12/dist-packages (from optuna) (25.0)\n", "Collecting sqlalchemy>=1.4.2 (from optuna)\n", " Downloading sqlalchemy-2.0.44-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (9.5 kB)\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.12/dist-packages (from optuna) (4.67.1)\n", "Requirement already satisfied: PyYAML in /usr/local/lib/python3.12/dist-packages (from optuna) (6.0.3)\n", "Collecting Mako (from alembic>=1.5.0->optuna)\n", " Downloading mako-1.3.10-py3-none-any.whl.metadata (2.9 kB)\n", "Requirement already satisfied: typing-extensions>=4.12 in /usr/local/lib/python3.12/dist-packages (from alembic>=1.5.0->optuna) (4.15.0)\n", "Collecting greenlet>=1 (from sqlalchemy>=1.4.2->optuna)\n", " Downloading greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl.metadata (4.1 kB)\n", "Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.12/dist-packages (from Mako->alembic>=1.5.0->optuna) (3.0.3)\n", "Downloading optuna-4.6.0-py3-none-any.whl (404 kB)\n", "Downloading alembic-1.17.1-py3-none-any.whl (247 kB)\n", "Downloading sqlalchemy-2.0.44-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m \u001b[33m0:00:00\u001b[0m\n", "\u001b[?25hDownloading greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl (607 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m607.6/607.6 kB\u001b[0m \u001b[31m91.8 MB/s\u001b[0m \u001b[33m0:00:00\u001b[0m\n", "\u001b[?25hDownloading colorlog-6.10.1-py3-none-any.whl (11 kB)\n", "Downloading mako-1.3.10-py3-none-any.whl (78 kB)\n", "Installing collected packages: Mako, greenlet, colorlog, sqlalchemy, alembic, optuna\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6/6\u001b[0m [optuna]2m5/6\u001b[0m [optuna]]my]\n", "\u001b[1A\u001b[2KSuccessfully installed Mako-1.3.10 alembic-1.17.1 colorlog-6.10.1 greenlet-3.2.4 optuna-4.6.0 sqlalchemy-2.0.44\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", "\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.3\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.12 -m pip install --upgrade pip\u001b[0m\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "pip install optuna" ] }, { "cell_type": "code", "execution_count": 6, "id": "fe66cb2c-72d2-4d31-a5af-58c48867c879", "metadata": {}, "outputs": [], "source": [ "%%capture\n", "import os, importlib.util\n", "!pip install --upgrade -qqq uv\n", "if importlib.util.find_spec(\"torch\") is None or \"COLAB_\" in \"\".join(os.environ.keys()):\n", " try: import numpy; get_numpy = f\"numpy=={numpy.__version__}\"\n", " except: get_numpy = \"numpy\"\n", " !uv pip install -qqq \\\n", " \"torch>=2.8.0\" \"triton>=3.4.0\" {get_numpy} torchvision bitsandbytes \"transformers==4.56.2\" trackio \\\n", " \"unsloth_zoo[base] @ git+https://github.com/unslothai/unsloth-zoo\" \\\n", " \"unsloth[base] @ git+https://github.com/unslothai/unsloth\" \\\n", " git+https://github.com/triton-lang/triton.git@05b2c186c1b6c9a08375389d5efe9cb4c401c075#subdirectory=python/triton_kernels\n", "elif importlib.util.find_spec(\"unsloth\") is None:\n", " !uv pip install -qqq unsloth trackio\n", "!uv pip install --upgrade --no-deps transformers==4.56.2 tokenizers trl==0.22.2 unsloth unsloth_zoo wandb" ] }, { "cell_type": "code", "execution_count": 3, "id": "c0709848-bba6-4daf-b1be-6c93a5990f91", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mjdmasciano2\u001b[0m (\u001b[33mjdmasciano2-university-of-lagos\u001b[0m) to \u001b[32mhttps://api.wandb.ai\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n" ] } ], "source": [ "!wandb login" ] }, { "cell_type": "code", "execution_count": 8, "id": "29bf54f0-46a9-45c3-9431-d364f11e153b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Overwriting fine_tune.py\n" ] } ], "source": [ "%%writefile fine_tune.py\n", "import os\n", "import json\n", "import torch\n", "import re\n", "from datasets import Dataset, DatasetDict\n", "from unsloth import FastLanguageModel\n", "from trl import SFTTrainer, SFTConfig\n", "import optuna # Import Optuna\n", "from unsloth.chat_templates import train_on_responses_only\n", "\n", "# --- 1. Define the Objective Function for Optuna ---\n", "def objective(trial):\n", " \"\"\"\n", " This function will be called by Optuna for each trial.\n", " It defines the hyperparameters to search, trains the model,\n", " and returns the evaluation loss which Optuna will aim to minimize.\n", " \"\"\"\n", " # --- A. Define the search space for hyperparameters based on the Unsloth guide ---\n", " learning_rate = trial.suggest_float(\"learning_rate\", 5e-6, 2e-4, log=True)\n", " lora_rank = trial.suggest_categorical(\"lora_rank\", [32, 64, 128])\n", " # The guide recommends lora_alpha = 2 * lora_rank. We derive it directly.\n", " lora_alpha = lora_rank * 2\n", " weight_decay = trial.suggest_float(\"weight_decay\", 0.0, 0.1)\n", "\n", " print(f\"\\n--- Starting Trial {trial.number} with parameters: ---\")\n", " print(f\" - learning_rate: {learning_rate:.2e}\")\n", " print(f\" - lora_rank: {lora_rank}\")\n", " print(f\" - lora_alpha: {lora_alpha}\")\n", " print(f\" - weight_decay: {weight_decay:.3f}\")\n", "\n", " # --- B. Model and Tokenizer Loading ---\n", " max_seq_length = 4096\n", " model, tokenizer = FastLanguageModel.from_pretrained(\n", " model_name=\"unsloth/gpt-oss-20b-BF16\",\n", " load_in_4bit=False, # Set to True if you need to save VRAM (QLoRA)\n", " max_seq_length=max_seq_length,\n", " )\n", "\n", " model = FastLanguageModel.get_peft_model(\n", " model,\n", " r=lora_rank, # From Optuna\n", " target_modules=[\n", " \"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n", " \"gate_proj\", \"up_proj\", \"down_proj\",\n", " ],\n", " lora_alpha=lora_alpha, # From Optuna\n", " use_gradient_checkpointing=\"unsloth\",\n", " random_state=3407,\n", " )\n", "\n", " # --- C. Dataset Loading and Preprocessing (remains the same) ---\n", " ROOT_DIR = \"/workspace/AIAC\"\n", " DATASET_FILE_PATH = os.path.join(ROOT_DIR, \"dipg_sft_.jsonl\")\n", " with open(DATASET_FILE_PATH, \"r\") as f:\n", " raw_data = [json.loads(line) for line in f if line.strip()]\n", " dataset = Dataset.from_list(raw_data)\n", " split_dataset = dataset.train_test_split(test_size=0.1, seed=42)\n", " dataset = DatasetDict({\"train\": split_dataset[\"train\"], \"test\": split_dataset[\"test\"]})\n", "\n", " def normalize_messages(messages):\n", " normalized = []\n", " for msg in messages:\n", " if msg[\"role\"] != \"assistant\":\n", " normalized.append(msg)\n", " continue\n", " content = msg[\"content\"]\n", " channels = re.findall(r\"<\\|channel\\|>(.*?)<\\|message\\|>(.*?)<\\|end\\|>\", content, re.DOTALL)\n", " if channels:\n", " thinking, final = \"\", \"\"\n", " for ch, text in channels:\n", " ch, text = ch.strip(), text.strip()\n", " if ch == \"analysis\": thinking += text + \"\\n\"\n", " elif ch == \"proof\": thinking += f\"\\n[Proof Section]\\n{text}\\n\"\n", " elif ch == \"final\": final += text\n", " normalized.append({\"role\": \"assistant\", \"thinking\": thinking.strip(), \"content\": final.strip()})\n", " else:\n", " normalized.append(msg)\n", " return normalized\n", "\n", " def formatting_prompts_func(examples):\n", " convos = [normalize_messages(convo) for convo in examples[\"messages\"]]\n", " return {\"text\": [tokenizer.apply_chat_template(convo, tokenize=False, add_generation_prompt=False) for convo in convos]}\n", "\n", " dataset = dataset.map(formatting_prompts_func, batched=True)\n", "\n", " # --- D. SFTTrainer with Dynamic Hyperparameters ---\n", " trainer = SFTTrainer(\n", " model=model,\n", " tokenizer=tokenizer,\n", " train_dataset=dataset['train'],\n", " eval_dataset=dataset['test'],\n", " args=SFTConfig(\n", " dataset_text_field=\"text\",\n", " per_device_train_batch_size=2, # Fixed based on your script\n", " gradient_accumulation_steps=4, # Fixed based on your script\n", " warmup_steps=10,\n", " max_seq_length=4096,\n", " max_steps=11, # Keep this low for a quick hyperparameter search\n", " learning_rate=learning_rate, # From Optuna\n", " logging_steps=5,\n", " optim=\"adamw_8bit\",\n", " weight_decay=weight_decay, # From Optuna\n", " lr_scheduler_type=\"linear\",\n", " seed=3407,\n", " eval_strategy=\"steps\",\n", " eval_steps=10,\n", " output_dir=f\"sft_outputs_trial_{trial.number}\", # Unique output dir\n", " report_to=\"wandb\",\n", " run_name=f\"trial-{trial.number}-lr-{learning_rate:.2e}-r-{lora_rank}\" # Descriptive W&B run name\n", " ),\n", " )\n", "\n", " # This part for training on responses only remains unchanged\n", " gpt_oss_kwargs = dict(instruction_part=\"<|start|>user<|message|>\", response_part=\"<|start|>assistant\")\n", " trainer = train_on_responses_only(trainer, **gpt_oss_kwargs)\n", "\n", " # --- E. Train and Evaluate ---\n", " print(f\"--- Starting SFT Training for Trial {trial.number} ---\")\n", " trainer.train()\n", " print(\"--- SFT Training Complete ---\")\n", "\n", " eval_results = trainer.evaluate()\n", " eval_loss = eval_results[\"eval_loss\"]\n", " print(f\"--- Trial {trial.number} finished with Eval Loss: {eval_loss} ---\")\n", " \n", " # Clean up to free VRAM for the next trial\n", " del model\n", " del trainer\n", " torch.cuda.empty_cache()\n", "\n", " return eval_loss\n", "\n", "# --- 2. Run the Hyperparameter Search ---\n", "if __name__ == \"__main__\":\n", " # Create a study object and specify the direction to optimize.\n", " study = optuna.create_study(direction=\"minimize\", study_name=\"unsloth_finetuning\")\n", " \n", " # Start the optimization. Optuna will call the 'objective' function 'n_trials' times.\n", " # Increase n_trials for a more thorough search (e.g., 20-50).\n", " study.optimize(objective, n_trials=10)\n", "\n", " print(\"\\n\\n--- Hyperparameter Search Complete ---\")\n", " print(\"Best trial:\")\n", " best_trial = study.best_trial\n", " \n", " print(f\" Value (min eval_loss): {best_trial.value}\")\n", " \n", " print(\" Best Parameters: \")\n", " for key, value in best_trial.params.items():\n", " print(f\" {key}: {value}\")\n", " \n", " # You can also get a dataframe with all trial results\n", " df = study.trials_dataframe()\n", " print(\"\\n--- All Trials ---\")\n", " print(df)" ] }, { "cell_type": "code", "execution_count": 9, "id": "26a0654b-2768-40d4-a9e0-c6001f163b8f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", "#### Unsloth: `hf_xet==1.1.10` and `ipykernel>6.30.1` breaks progress bars. Disabling for now in XET.\n", "#### Unsloth: To re-enable progress bars, please downgrade to `ipykernel==6.30.1` or wait for a fix to\n", "https://github.com/huggingface/xet-core/issues/526\n", "INFO 11-13 15:12:29 [__init__.py:225] Automatically detected platform rocm.\n", "🦥 Unsloth Zoo will now patch everything to make training faster!\n", "\u001b[32m[I 2025-11-13 15:12:33,207]\u001b[0m A new study created in memory with name: unsloth_finetuning\u001b[0m\n", "\n", "--- Starting Trial 0 with parameters: ---\n", " - learning_rate: 2.09e-05\n", " - lora_rank: 128\n", " - lora_alpha: 256\n", " - weight_decay: 0.039\n", "Unsloth: AMD currently is not stable with 4bit bitsandbytes. Disabling for now.\n", "Unsloth: AMD currently is not stable with 4bit bitsandbytes. Disabling for now.\n", "==((====))== Unsloth 2025.10.9: Fast Gpt_Oss patching. Transformers: 4.56.2. vLLM: 0.11.1rc3.dev39+gf417746ad.rocm700.\n", " \\\\ /| . Num GPUs = 1. Max memory: 191.688 GB. Platform: Linux.\n", "O^O/ \\_/ \\ Torch: 2.9.0a0+git1c57644. ROCm Toolkit: 7.0.51831-a3e329ad8. Triton: 3.4.0\n", "\\ / Bfloat16 = TRUE. FA [Xformers = None. FA2 = True]\n", " \"-____-\" Free license: http://github.com/unslothai/unsloth\n", "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n", "Loading checkpoint shards: 100%|██████████████████| 9/9 [00:19<00:00, 2.19s/it]\n", "Unsloth: Making `model.base_model.model.model` require gradients\n", "Map: 100%|███████████████████████████| 900/900 [00:00<00:00, 6437.47 examples/s]\n", "Map: 100%|███████████████████████████| 100/100 [00:00<00:00, 6146.85 examples/s]\n", "Unsloth: Tokenizing [\"text\"] (num_proc=24): 100%|█| 900/900 [00:10<00:00, 83.14 \n", "Unsloth: Tokenizing [\"text\"] (num_proc=24): 100%|█| 100/100 [00:08<00:00, 11.47 \n", "Map (num_proc=24): 100%|██████████████| 900/900 [00:01<00:00, 849.22 examples/s]\n", "Map (num_proc=24): 100%|██████████████| 100/100 [00:00<00:00, 108.45 examples/s]\n", "--- Starting SFT Training for Trial 0 ---\n", "The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': 199998}.\n", "==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 1\n", " \\\\ /| Num examples = 900 | Num Epochs = 1 | Total steps = 11\n", "O^O/ \\_/ \\ Batch size per device = 2 | Gradient accumulation steps = 4\n", "\\ / Data Parallel GPUs = 1 | Total batch size (2 x 4 x 1) = 8\n", " \"-____-\" Trainable parameters = 63,700,992 of 20,978,458,176 (0.30% trained)\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mjdmasciano2\u001b[0m (\u001b[33mjdmasciano2-university-of-lagos\u001b[0m) to \u001b[32mhttps://api.wandb.ai\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[38;5;178m⢿\u001b[0m Waiting for wandb.init()...\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[38;5;178m⣻\u001b[0m Waiting for wandb.init()...\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.23.0\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/workspace/AIAC/OpenEnv/wandb/run-20251113_151330-z9cgpnww\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mtrial-0-lr-2.09e-05-r-128\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/jdmasciano2-university-of-lagos/huggingface\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/jdmasciano2-university-of-lagos/huggingface/runs/z9cgpnww\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Detected [huggingface_hub.inference, openai] in use.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: For more information, check out the docs at: https://weave-docs.wandb.ai/\n", " 0%| | 0/11 [00:00(.*?)<\\|message\\|>(.*?)<\\|end\\|>\", content, re.DOTALL)\n", " if channels:\n", " thinking, final = \"\", \"\"\n", " for ch, text in channels:\n", " ch, text = ch.strip(), text.strip()\n", " if ch == \"analysis\": thinking += text + \"\\n\"\n", " elif ch == \"proof\": thinking += f\"\\n[Proof Section]\\n{text}\\n\"\n", " elif ch == \"final\": final += text\n", " normalized.append({\"role\": \"assistant\", \"thinking\": thinking.strip(), \"content\": final.strip()})\n", " else:\n", " normalized.append(msg)\n", " return normalized\n", "\n", " def formatting_prompts_func(examples):\n", " convos = [normalize_messages(convo) for convo in examples[\"messages\"]]\n", " return {\"text\": [tokenizer.apply_chat_template(convo, tokenize=False, add_generation_prompt=False) for convo in convos]}\n", "\n", " dataset = dataset.map(formatting_prompts_func, batched=True)\n", "\n", " # --- D. SFTTrainer with Dynamic Hyperparameters ---\n", " trainer = SFTTrainer(\n", " model=model,\n", " tokenizer=tokenizer,\n", " train_dataset=dataset['train'],\n", " eval_dataset=dataset['test'],\n", " args=SFTConfig(\n", " dataset_text_field=\"text\",\n", " per_device_train_batch_size=2, # Fixed based on your script\n", " gradient_accumulation_steps=4, # Fixed based on your script\n", " warmup_steps=10,\n", " max_seq_length=4096,\n", " max_steps=11, # Keep this low for a quick hyperparameter search\n", " learning_rate=learning_rate, # From Optuna\n", " logging_steps=5,\n", " optim=\"adamw_8bit\",\n", " weight_decay=weight_decay, # From Optuna\n", " lr_scheduler_type=\"linear\",\n", " seed=3407,\n", " eval_strategy=\"steps\",\n", " eval_steps=10,\n", " output_dir=f\"sft_outputs_trial_{trial.number}\", # Unique output dir\n", " report_to=\"wandb\",\n", " run_name=f\"trial-{trial.number}-lr-{learning_rate:.2e}-r-{lora_rank}\" # Descriptive W&B run name\n", " ),\n", " )\n", "\n", " # This part for training on responses only remains unchanged\n", " gpt_oss_kwargs = dict(instruction_part=\"<|start|>user<|message|>\", response_part=\"<|start|>assistant\")\n", " trainer = train_on_responses_only(trainer, **gpt_oss_kwargs)\n", "\n", " # --- E. Train and Evaluate ---\n", " print(f\"--- Starting SFT Training for Trial {trial.number} ---\")\n", " trainer.train()\n", " print(\"--- SFT Training Complete ---\")\n", "\n", " eval_results = trainer.evaluate()\n", " eval_loss = eval_results[\"eval_loss\"]\n", " print(f\"--- Trial {trial.number} finished with Eval Loss: {eval_loss} ---\")\n", " \n", " # Clean up to free VRAM for the next trial\n", " del model\n", " del trainer\n", " torch.cuda.empty_cache()\n", "\n", " return eval_loss\n", "\n", "# --- 2. Run the Hyperparameter Search ---\n", "if __name__ == \"__main__\":\n", " # Create a study object and specify the direction to optimize.\n", " study = optuna.create_study(direction=\"minimize\", study_name=\"unsloth_finetuning_l_r\")\n", " \n", " # Start the optimization. Optuna will call the 'objective' function 'n_trials' times.\n", " # Increase n_trials for a more thorough search (e.g., 20-50).\n", " study.optimize(objective, n_trials=10)\n", "\n", " print(\"\\n\\n--- Hyperparameter Search Complete ---\")\n", " print(\"Best trial:\")\n", " best_trial = study.best_trial\n", " \n", " print(f\" Value (min eval_loss): {best_trial.value}\")\n", " \n", " print(\" Best Parameters: \")\n", " for key, value in best_trial.params.items():\n", " print(f\" {key}: {value}\")\n", " \n", " # You can also get a dataframe with all trial results\n", " df = study.trials_dataframe()\n", " print(\"\\n--- All Trials ---\")\n", " print(df)" ] }, { "cell_type": "code", "execution_count": 11, "id": "2b646549-f498-4524-a1d9-1951ff6831da", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", "#### Unsloth: `hf_xet==1.1.10` and `ipykernel>6.30.1` breaks progress bars. Disabling for now in XET.\n", "#### Unsloth: To re-enable progress bars, please downgrade to `ipykernel==6.30.1` or wait for a fix to\n", "https://github.com/huggingface/xet-core/issues/526\n", "INFO 11-13 15:53:42 [__init__.py:225] Automatically detected platform rocm.\n", "🦥 Unsloth Zoo will now patch everything to make training faster!\n", "\u001b[32m[I 2025-11-13 15:53:45,819]\u001b[0m A new study created in memory with name: unsloth_finetuning_l_r\u001b[0m\n", "\n", "--- Starting Trial 0 with parameters: ---\n", " - learning_rate: 1.15e-05\n", " - lora_rank: 128\n", " - lora_alpha: 128\n", " - weight_decay: 0.083\n", "Unsloth: AMD currently is not stable with 4bit bitsandbytes. Disabling for now.\n", "Unsloth: AMD currently is not stable with 4bit bitsandbytes. Disabling for now.\n", "==((====))== Unsloth 2025.10.9: Fast Gpt_Oss patching. Transformers: 4.56.2. vLLM: 0.11.1rc3.dev39+gf417746ad.rocm700.\n", " \\\\ /| . Num GPUs = 1. Max memory: 191.688 GB. Platform: Linux.\n", "O^O/ \\_/ \\ Torch: 2.9.0a0+git1c57644. ROCm Toolkit: 7.0.51831-a3e329ad8. Triton: 3.4.0\n", "\\ / Bfloat16 = TRUE. FA [Xformers = None. FA2 = True]\n", " \"-____-\" Free license: http://github.com/unslothai/unsloth\n", "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n", "Loading checkpoint shards: 100%|██████████████████| 9/9 [00:15<00:00, 1.74s/it]\n", "Unsloth: Making `model.base_model.model.model` require gradients\n", "Map: 100%|███████████████████████████| 900/900 [00:00<00:00, 8761.46 examples/s]\n", "Map: 100%|███████████████████████████| 100/100 [00:00<00:00, 9272.87 examples/s]\n", "Unsloth: Tokenizing [\"text\"] (num_proc=24): 100%|█| 900/900 [00:07<00:00, 112.84\n", "Unsloth: Tokenizing [\"text\"] (num_proc=24): 100%|█| 100/100 [00:05<00:00, 18.24 \n", "Map (num_proc=24): 100%|█████████████| 900/900 [00:00<00:00, 1166.64 examples/s]\n", "Map (num_proc=24): 100%|██████████████| 100/100 [00:00<00:00, 167.06 examples/s]\n", "--- Starting SFT Training for Trial 0 ---\n", "The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': 199998}.\n", "==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 1\n", " \\\\ /| Num examples = 900 | Num Epochs = 1 | Total steps = 11\n", "O^O/ \\_/ \\ Batch size per device = 2 | Gradient accumulation steps = 4\n", "\\ / Data Parallel GPUs = 1 | Total batch size (2 x 4 x 1) = 8\n", " \"-____-\" Trainable parameters = 63,700,992 of 20,978,458,176 (0.30% trained)\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mjdmasciano2\u001b[0m (\u001b[33mjdmasciano2-university-of-lagos\u001b[0m) to \u001b[32mhttps://api.wandb.ai\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[38;5;178m⢿\u001b[0m Waiting for wandb.init()...\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[38;5;178m⣻\u001b[0m Waiting for wandb.init()...\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.23.0\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/workspace/AIAC/OpenEnv/wandb/run-20251113_155424-j2kz7jrl\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mtrial-0-lr-1.15e-05-r-128\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/jdmasciano2-university-of-lagos/huggingface\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/jdmasciano2-university-of-lagos/huggingface/runs/j2kz7jrl\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Detected [huggingface_hub.inference, openai] in use.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: For more information, check out the docs at: https://weave-docs.wandb.ai/\n", " 0%| | 0/11 [00:00