{ "cells": [ { "cell_type": "code", "execution_count": 5, "id": "db4208b9-5da4-46df-b77a-0f1836c9e4ec", "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\" # force using CUDA device 1\n", "os.environ[\"ZE_AFFINITY_MASK\"] = \"1\" # force using Intel XPU device 1\n", "from peft import PeftConfig, PeftModel\n", "from peft import PeftModel, PeftConfig\n", "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n", "from datasets import load_dataset\n", "import torch\n", "import random\n", "\n", "peft_model_id = \"smangrul/tinyllama_lora_norobots\"\n", "device = torch.accelerator.current_accelerator().type if hasattr(torch, \"accelerator\") else \"cuda\"\n", "config = PeftConfig.from_pretrained(peft_model_id)\n", "model_kwargs = {\"device_map\": \"auto\"}\n", "model_kwargs[\"quantization_config\"] = BitsAndBytesConfig(load_in_4bit=True)\n", "model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, **model_kwargs)\n", "tokenizer = AutoTokenizer.from_pretrained(peft_model_id)\n", "model.resize_token_embeddings(len(tokenizer))\n", "model = PeftModel.from_pretrained(model, peft_model_id, adapter_name=\"norobots\")\n", "_ = model.load_adapter(\"smangrul/tinyllama_lora_sql\", adapter_name=\"sql\")\n", "_ = model.load_adapter(\"smangrul/tinyllama_lora_adcopy\", adapter_name=\"adcopy\")" ] }, { "cell_type": "code", "execution_count": 6, "id": "541dab43-9675-42a2-8d90-7437df9f0fa0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 17.1 s, sys: 458 ms, total: 17.5 s\n", "Wall time: 1.94 s\n" ] } ], "source": [ "%%time\n", "# [0.8, 0.1, 0.1] linear #[1.0, 0.2] 0.7 density dare_linear #[1.5, 0.3] 0.5 density ties #[0.8, 0.5] cat\n", "adapters = [\"norobots\", \"adcopy\", \"sql\"]\n", "weights = [2.0, 0.3, 0.7]\n", "adapter_name = \"merge\"\n", "density = 0.2\n", "combination_type = \"ties\"\n", "if adapter_name in model.peft_config:\n", " model.delete_adapter(adapter_name)\n", "model.add_weighted_adapter(adapters, weights, adapter_name, combination_type=combination_type, density=density)" ] }, { "cell_type": "code", "execution_count": 7, "id": "76596671-3677-47f0-9d66-81f40bc4d726", "metadata": {}, "outputs": [], "source": [ "model.eval()\n", "model.set_adapter(\"merge\")" ] }, { "cell_type": "code", "execution_count": null, "id": "9d59f9f3-6313-43d8-be36-4ca2bbb105b2", "metadata": {}, "outputs": [], "source": [ "messages = [\n", " {\"role\": \"user\", \"content\": \"Write an essay about Generative AI.\"},\n", "]\n", "text = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)\n", "inputs = tokenizer(text, return_tensors=\"pt\") # , add_special_tokens=False)\n", "inputs = {k: v.to(device) for k, v in inputs.items()}\n", "outputs = model.generate(\n", " **inputs,\n", " max_new_tokens=256,\n", " do_sample=True,\n", " top_p=0.95,\n", " temperature=0.2,\n", " repetition_penalty=1.2,\n", " eos_token_id=tokenizer.eos_token_id,\n", ")\n", "print(tokenizer.decode(outputs[0]))" ] }, { "cell_type": "code", "execution_count": null, "id": "e5c1daeb-59c8-41d7-bebb-7abd052ab917", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "<|im_start|>system \n", "Create a text ad given the following product and description.<|im_end|> \n", "<|im_start|>user \n", "Product: Sony PS5 PlayStation Console\n", "Description: The PS5™ console unleashes new gaming possibilities that you never anticipated.<|im_end|> \n", "<|im_start|>assistant \n", "Ad Text: Experience the next-gen power of the all-new Sony PS5 with its stunning visuals, innovative gameplay features, and more! Get ready to play in style as you experience the future of gaming on your own terms.<|im_end|>\n" ] } ], "source": [ "messages = [\n", " {\"role\": \"system\", \"content\": \"Create a text ad given the following product and description.\"},\n", " {\n", " \"role\": \"user\",\n", " \"content\": \"Product: Sony PS5 PlayStation Console\\nDescription: The PS5™ console unleashes new gaming possibilities that you never anticipated.\",\n", " },\n", "]\n", "text = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)\n", "inputs = tokenizer(text, return_tensors=\"pt\") # , add_special_tokens=False)\n", "inputs = {k: v.to(device) for k, v in inputs.items()}\n", "outputs = model.generate(\n", " **inputs,\n", " max_new_tokens=128,\n", " do_sample=True,\n", " top_p=0.95,\n", " temperature=0.2,\n", " repetition_penalty=1.2,\n", " eos_token_id=tokenizer.eos_token_id,\n", ")\n", "print(tokenizer.decode(outputs[0]))" ] }, { "cell_type": "code", "execution_count": null, "id": "5bb08b46-90ae-48a8-8783-ca74b3e26e42", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Table: 2-11365528-2\n", "Columns: ['Team', 'Head Coach', 'President', 'Home Ground', 'Location']\n", "Natural Query: Who is the Head Coach of the team whose President is Mario Volarevic?\n", "SQL Query: SELECT Head Coach FROM 2-11365528-2 WHERE President = Mario Volarevic\n" ] } ], "source": [ "text = \"\"\"Table: 2-11365528-2\n", "Columns: ['Team', 'Head Coach', 'President', 'Home Ground', 'Location']\n", "Natural Query: Who is the Head Coach of the team whose President is Mario Volarevic?\n", "SQL Query:\"\"\"\n", "\n", "inputs = tokenizer(text, return_tensors=\"pt\") # , add_special_tokens=False)\n", "inputs = {k: v.to(device) for k, v in inputs.items()}\n", "outputs = model.generate(\n", " **inputs, max_new_tokens=64, repetition_penalty=1.1, eos_token_id=tokenizer(\"\").input_ids[-1]\n", ")\n", "print(tokenizer.decode(outputs[0]))" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.13" } }, "nbformat": 4, "nbformat_minor": 5 }