{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "This notebook shows how to use the adapter merging methods from `peft` and apply them image generation models using `diffusers`." ] }, { "cell_type": "markdown", "metadata": { "id": "QaEZ3dPgGtza" }, "source": [ "## Turn `diffusers` LoRA checkpoints into `PeftModel`" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "KwvBjN-e62ts", "outputId": "19c58dc7-95db-49c6-beb8-3ade1a8fe284" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/1.9 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r", "\u001b[2K \u001b[91m━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.2/1.9 MB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 MB\u001b[0m \u001b[31m32.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m280.0/280.0 kB\u001b[0m \u001b[31m34.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.4/8.4 MB\u001b[0m \u001b[31m100.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for peft (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n" ] } ], "source": [ "!pip install diffusers accelerate transformers -U -q\n", "!pip install git+https://github.com/huggingface/peft -q" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5S64sUQhJqB3" }, "outputs": [], "source": [ "from google.colab import userdata\n", "TOKEN = userdata.get(\"HF_TOKEN\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 150, "referenced_widgets": [ "c9f764b5036042af9f1505e3729cbc32", "b999b3e3af3744a79c0c90657ef37d4e", "f3baef4fbf4b4ec08480522be921f841", "bd2740e191a74558a77a965b4f2d7f28", "6f6a6cfd50404f1ea09f83e95b04550a", "713dec1904ce46f6b2d5a9b7e3e0373a", "a6bb8206de044c74a03d1a64c801e742", "f2c67c29e1224df3b2def5a87eb8d368", "486282a4ead148868005c592d74a4ed4", "48f51c96b7574946bf9542633eb39135", "35c810f4bfe741f091f172cede413950", "67567eea233b423c8acd62773a4adb30", "5161248cd0384d5887ed231ecd48c82e", "6f72ea2c284e4e40899375c7b07c517f", "d99a364420454ba5bfd510d1226b94af", "d33c90e8ea0945d397659f6a90cf51b6", "097fa44254ea4ecda7c8db995f370afc", "9fb982789fbc4582bceb356e351db438", "0204153b17ca4bb0b21fc033393ce9bd", "f02041b1d5e1485bb2ba02b00fc2c242", "4d327c9e91b34c7b84cedd8f9660e9fd", "b44f7154c55146a3bf5f4bd9e438086f", "b27ac2aaff694dd5999ab2cba91195da", "556730e12d5d4e0ea51a0dd1b5aac331", "584dd148b2344bdc92f1d0850399aed7", "ccc5bdc185a84901994577ff7f1bc962", "2c75632d8fdc4458814055172d1d72c3", "2f016774e7854ef589442734d0bb2f08", "a15c3a32bfd347a98c2a50d27cd5b9f9", "67ef71b6521b47dd90e3dc0fd03016f2", "989c54778eb7469fb91e4337d6f49b0b", "439cf6c6f9e845cea4008e3219454ff4", "460c3c96d1724713b78bddcfc1f3eb97" ] }, "id": "1YH9xWDcyhaa", "outputId": "8b7b32f4-4b77-411e-f499-7b2cf7650613" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "The cache for model files in Transformers v4.22.0 has been updated. Migrating your old cache. This is a one-time only operation. You can interrupt this and resume the migration later on by calling `transformers.utils.move_cache()`.\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "c9f764b5036042af9f1505e3729cbc32", "version_major": 2, "version_minor": 0 }, "text/plain": [ "0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "67567eea233b423c8acd62773a4adb30", "version_major": 2, "version_minor": 0 }, "text/plain": [ "unet/config.json: 0%| | 0.00/1.68k [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b27ac2aaff694dd5999ab2cba91195da", "version_major": 2, "version_minor": 0 }, "text/plain": [ "diffusion_pytorch_model.fp16.safetensors: 0%| | 0.00/5.14G [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from diffusers import UNet2DConditionModel\n", "import torch\n", "\n", "model_id = \"stabilityai/stable-diffusion-xl-base-1.0\"\n", "unet = UNet2DConditionModel.from_pretrained(\n", " model_id, subfolder=\"unet\", torch_dtype=torch.float16, use_safetensors=True, variant=\"fp16\"\n", ").to(\"cuda\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "-kfTBaLR1Mp-" }, "outputs": [], "source": [ "# So that we can populate it later.\n", "import copy\n", "\n", "sdxl_unet = copy.deepcopy(unet)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 593, "referenced_widgets": [ "0b7ffee735044ece90f010c6771d2d69", "27671f87d0f3400b89e34bc428daa53c", "33a018bf2ed547caa69f8ca91dcbb112", "a91df5ceea8647d98f4d49701bb00969", "c8555cb30ade40b9bd9a3435d6deeb67", "b28a2b59aab9478e9ac3ab24a1de6f9a", "c08fca70486040069d4f8f1df46a1074", "bab2458ba3f54b229828ad9c8706aba4", "2a0f746d6eab4680b6c44ec5cdbd8fd9", "5269d405725946b3a657d3a8d7b25885", "b4a697d8335c435ab21219716a1da022", "85f22e78d47641d19efb6e6d62f6a014", "d8bbb7402f3e44b2899fc98f02cee87e", "8b41763280a048d485f06682ddc12ca2", "05791b70a24a42138755141602399c47", "753f64d9069640d985f399a058fc9b5b", "19679186751b42e3ad2c44ea46c82a9c", "d4baf7891f854c9c9898314633f97356", "9c6ab017f5fb46eda8e3c22e9dd2b838", "10bc970e474a46249e2a5e9e43fec7be", "5c838962c7854541b61877c6d42481c4", "4d54c07dd8e94557b82521772c1a2825", "efd33121bdfc4ce195a07c9ef523a477", "d2cd2572973249baa8e130b5777f4147", "4e77d8b6cdb94fc68974d27b24cfb3fd", "c292a598350d4dd4bb9b70aab1320c29", "c68a5ad3bb664e8785e724085d208e96", "b27c74f0c7bf493fa8bcb4c5b9c9c100", "09638c8da74f4dddaf1d5d94dd8ad885", "6a8a634cd3844fa081a4622d224ec940", "2460b9e05b58481e898b139b94532c14", "cdd8f9b1592842b48e1ffa80f8ec8246", "ba9b002888b448738ba4b127e1046f5d", "54ab158c643540abb8b3a96c1ae3ecab", "55130e37444844f88286a87c8d153eba", "7e3048e0fac94dfdaca1bc862ecfce15", "643ba607008547ba9572fa4880a7d0f2", "5738e5d103254c5485856f97d82954ab", "beedc3b275c24432a2e959dcf9dd418d", "10690b309908402289e9891203714199", "0c68c6c65fb94650b15069a25d9e1699", "ad233dfd52034b9e8c6c2c5b86995717", "2ef25bd6dd644347baac12366d7002fd", "0918b23f41e0404e82fccd08cadc6ccf", "b25e822b3f77431fb72b4780067c90d9", "55177afb435f44898df300143703e4d8", "b79d3f5bd8024451bc7148ba2a5029bd", "eacd646e2b984e60ab603bfc6d631de8", "d158a80f4186411a8a9c335a8d5a888e", "7049676db714446b98bba16b5f1b049e", "09172ff4be4e433483d27b576464d1df", "ea257c1c73524141b87ab3c1ef85c908", "c474ed5e340146baae6c38f62013afe3", "2ece04bc10934b3cb9d383abfc5ccd6e", "67a03631ebc54f99928f0feb18ab38af", "40ff502d2d5c40378c25cf84dd3323c8", "230bd59922e84b90b0a141b3ac1e681b", "6d0f946764444df28cb0da0fd0a408eb", "c995197e66e04874a9f5d34db98b8890", "f2596717405c40e1a39b721386e7a972", "f8e6babf4fdd4c8e80d6dd24ff22d464", "5e030e8a026b4513aa954203169f0a27", "f9ca5d4810b34938b6f997ff66a8d541", "b418ff1733db4efbab1b00b632b894e2", "83f2d1dfaba54da38f0421b69930c3c1", "577cc4b4f27941189c62951046db24ec", "a16016987b6145b69caaac6712d72835", "c82cf8cf90ed4e93bccffcf75881a56d", "5a4c6dc09a1049c0adfca9834b045a25", "c4d83c2b37504473afe63209a178b4cd", "fae9d16daace412492b048b012b8d6dc", "421b59f4021d4c4d930c51b6b4c7071f", "60cc7415644d4e16b88f8fc5896b4b3b", "fd13a58d6b444a0f955832647f64df12", "8d492d53eb7f4225b516a65ef80f24e5", "70849993c0c94ecf87c56e430f06181d", "e820c557697648378966ed0a073826c8", "a9ac3b2188594c19885bcdb9a659ecde", "a708db5805424449afa269b714ebb3b3", "6d492ddabcbe4d65b5d311834865ab92", "e17e3253c16743a29f09b82d23c3b26d", "e67a69c294334b01974f8bef36f133a0", "24cbd9338ad94801aa11f4dcb2a867cb", "c88fcf0d20154b1fb9b7e8a00116b5e6", "ee8407365f5d42d9b98536152c9efe92", "5ceb5c369f974b84ba850c9e81730a0e", "a7767d5a440f4c819cdb87414f2187ff", "f0bc6b14a299445ca705b888b3047064", "fd0ff16b68b2488d8c31ebe700dee9c9", "0e49d820da754da785bec2e5940eb9f6", "0b53b908088648e3b2beadaeba0f5da1", "804e7ee768794bba88aec3137f418868", "0d786d8386ba49d0b53a5452d52e722d", "33d459c0fbfe45389abe7eb43d2710a1", "dd42a2ff90854b74ba5fde1de26b4e15", "a3ce5829f8a640e79a19d737663b8474", "62afb52a01924566b52f6c2d9ffb76f4", "7a52c380a91c4f49bbfde658550248d3", "3e418e46c47841ac9d717a6981807f68", "8840288388ae4162884feef9c8e776f7", "67b7783351d340d99c44149635b9be84", "0f60c0f123954744ad13b670ca6dce77", "dfabe7aa70024d1aa868ef5e6650dc6d", "7d282fda276343c3aff99b001253ecc1", "4cae160456c74a5fa761f026d64b2e35", "819a26acd18f443882feb129f2c576d6", "d2ed1988cf8c4bcdb1793b3d5068efed", "6f113303101b4f448380a878d8900bf6", "60e8e9f8e6ce40dd910ce1a9410b5e24", "6703a0417c474db5bf261fe8679051e9", "cb89ecd5a8c14051985495da1797a202", "61db57127c5845759360bdf8b29dac2d", "ab9c869439a94bddbbc0c6098f4c5b2a", "587bd0b90afa450ba82e49cf86ee135d", "955ec2551f8b400dbbdba68d7449de76", "f46df85f441e4ada831f0e2b142f296a", "d4f1dbe4ce244abc987b1089876e080f", "13ac509e0d5f42e9bbb6deda62f77923", "4a4a50a17c014f189b52119901104d79", "cf42fc299989442f94f4a9df63005ab4", "90b9300ba00e41e58170ec5634622985", "4fa4f545258b4d6ca014a4c84ec4b24a", "993e5303f107468f83dbf51026e64301", "1e6c1c848e364ce4841ddcaa1383cfba", "d432098a941c463599648b156abea24b", "917139ad07a64e0cae89f2beeffae956", "83afce4becac4f37ba916bf1901346ff", "071e40d45ad14fc19b1480927d15d2ae", "fe19dcea6d9a44d28f077e065f1671c4", "08fddacdff4b4ae09adb5440fd86ae86", "d62465d39d7e4265832901e9b9707993", "4f56214f69034077bafdfdabc1c2aebf", "29b67774c6944bc7ab93e7ba0eaf867f", "0805d34df44f451d9b9910dbe5999245", "2f2c0ea0fc914e7981e34d01751f74d8", "5f52ae61812544f29f774f80fcb7a09c", "ca5dc8342ef946a49d9e67a21f1a67c8", "babce0e85baf4279ae1d22d64006667f", "5cf6cd09cda64d688f4a0f7c511533d0", "f668dd13af6f41d8be358f7db5261c54", "8a8ef60b3b72452fb9ccc31052ab3b4f", "7d29b20296aa4e33837b8ad53fc4adba", "6a94d1d176b844db96de0c0e3cd67701", "c80901426c87439481078b9da2e0c772", "416f31db79fd431fb8dc06e994421b70", "1971fcd35a564c449b4f437dac46058f", "76a474e4aab14a5987cf25d1391d578c", "ff3bf3f1873c4b01b0a547dfe02923ce", "a6295c7e7630444c9b7425b884ad9707", "3ede828a0374453e9aac9a6695befae3", "dc4391fe30694a788134fffdd2a23d1a", "cfc6ec59d45d42d187f42061902abbfb", "68822f8f861a48859df630fad63f51c1", "2671327d35e64f0da670a3a611fd0886", "aae194e7f71f4eba80ffd18f91f083a8", "9ad4192f6f244264aede1b3c8ac3c57a", "90558769855c4bb08ff2fe4af940c45b", "75384779c5d540d28afb53a0e318b674", "a287f197e4fe4f908e3ee0a0e6cb35cf", "fdb799739700447d8a5198f1f4f9b17f", "683db0ff105a47038c350cfc74b88345", "b79d32186443469d94836b663bf156b5", "86cb336c1f684867bd13dae0370b4d36", "7f63b3d80ecb4c3eb837bd0e616e1623", "4f6e7b21ba0747f19b9d63468861c988", "766567be45eb4db6b8d7a3364566c1dc", "3c222a56d4404863a1dac60f1d03835b", "db862ffbb44d450db514173df4c7f301", "ddbeb13bb8174fc0b7d5543108d1c4f5", "5b3bbc663d504fb99318ba186e6b8499", "30bfac68f4224152b048d6ccf6013c5c", "971f59d5f3e04697b3ab3c39ec0fc667", "fe958df746be4dc1871bd58628697c3c", "54417d8b9c5249d0a028d9e831dd8be6", "9a37ffb9810f482b80f245f64947c371", "bdfe4e4109a14a41bcd2e1e4242d82bb", "22e894ba852e4072a1f63a83b3a98b16", "95e24bbc8397455fabb724ed3c330511", "d35b3848508c4b6390c243866649439d", "c7cf10df8d7944aeb93947d5f4156c92", "8e3b0b9f26a34ab8a932756b32834fd0", "b1d427082b9f452eb6546c7d55016b36", "061fcc6e5f3c44c48cde212c1ba515e5", "d70508c304794bc79e80aab136eaf65a", "01b20535e40b47068723073ac6c819ee", "de0b54a59d9f47408d92915ad746cd5e", "eb9a5f255fa0447eba6a33e1c30ba166", "d1e4d2fd70e644f986104c998db7e53b", "f786a0f386f6486083c15e576f6eb3e7", "6715d91c6b62426aa49c344b65bcd8a2", "0b0a85e1133c4ca3ba716e2403511703", "017dc44dbe1c4de491e003a1e279a218", "e3a1a5e9f29d4d28b0b9496493dafa21", "c008577d922e436aabb8680ca0d13117", "bd176c410a4e48a382a1b688e77aafcb", "6b586ad7a3054c83b14243d69484127f", "34fcd527a59748cf832c9efdb954fc0e", "fd672cd5ba0c4695be4240707dcf4bf3" ] }, "id": "EMTVH9cLEZYi", "outputId": "7a24b4b0-71f6-4c65-a242-fb3d502da6a8" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "0b7ffee735044ece90f010c6771d2d69", "version_major": 2, "version_minor": 0 }, "text/plain": [ "model_index.json: 0%| | 0.00/609 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "85f22e78d47641d19efb6e6d62f6a014", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Fetching 17 files: 0%| | 0/17 [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "efd33121bdfc4ce195a07c9ef523a477", "version_major": 2, "version_minor": 0 }, "text/plain": [ "model.fp16.safetensors: 0%| | 0.00/246M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "54ab158c643540abb8b3a96c1ae3ecab", "version_major": 2, "version_minor": 0 }, "text/plain": [ "scheduler/scheduler_config.json: 0%| | 0.00/479 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b25e822b3f77431fb72b4780067c90d9", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer/tokenizer_config.json: 0%| | 0.00/737 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "40ff502d2d5c40378c25cf84dd3323c8", "version_major": 2, "version_minor": 0 }, "text/plain": [ "text_encoder_2/config.json: 0%| | 0.00/575 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "a16016987b6145b69caaac6712d72835", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer/special_tokens_map.json: 0%| | 0.00/472 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "a9ac3b2188594c19885bcdb9a659ecde", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer/merges.txt: 0%| | 0.00/525k [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "fd0ff16b68b2488d8c31ebe700dee9c9", "version_major": 2, "version_minor": 0 }, "text/plain": [ "text_encoder/config.json: 0%| | 0.00/565 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "8840288388ae4162884feef9c8e776f7", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer/vocab.json: 0%| | 0.00/1.06M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "cb89ecd5a8c14051985495da1797a202", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer_2/tokenizer_config.json: 0%| | 0.00/725 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4fa4f545258b4d6ca014a4c84ec4b24a", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer_2/special_tokens_map.json: 0%| | 0.00/460 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "29b67774c6944bc7ab93e7ba0eaf867f", "version_major": 2, "version_minor": 0 }, "text/plain": [ "vae/config.json: 0%| | 0.00/642 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "c80901426c87439481078b9da2e0c772", "version_major": 2, "version_minor": 0 }, "text/plain": [ "model.fp16.safetensors: 0%| | 0.00/1.39G [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "aae194e7f71f4eba80ffd18f91f083a8", "version_major": 2, "version_minor": 0 }, "text/plain": [ "diffusion_pytorch_model.fp16.safetensors: 0%| | 0.00/167M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "766567be45eb4db6b8d7a3364566c1dc", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer_2/vocab.json: 0%| | 0.00/1.06M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "22e894ba852e4072a1f63a83b3a98b16", "version_major": 2, "version_minor": 0 }, "text/plain": [ "diffusion_pytorch_model.fp16.safetensors: 0%| | 0.00/167M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "d1e4d2fd70e644f986104c998db7e53b", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading pipeline components...: 0%| | 0/7 [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Load the pipeline too.\n", "from diffusers import DiffusionPipeline\n", "\n", "pipe = DiffusionPipeline.from_pretrained(\n", " model_id, variant=\"fp16\", torch_dtype=torch.float16, unet=unet\n", ").to(\"cuda\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 49, "referenced_widgets": [ "5e677518051c43768dbf06243701c817", "d727913663634e368ade4a7dc64fe74b", "eb73095c804a4272856fe348fa3cb1e9", "7e9b46b10fa24dfea489dfbc150d2a2e", "c8156b0cc68e4b3693dcabc530a4ea9a", "d987907f09084d44b452f939aadff65e", "e976b994189343f5ba7d762ef92c79e2", "9810873713024e79ae6d338dfeae5876", "2e18ce21c01a4a3ca992622957e7d297", "2db944a049a04426bba181fddb2801b1", "683863a313034025ab99fab5810f39c7" ] }, "id": "D5hL5156zPis", "outputId": "2510b8e7-c030-40f8-dcd2-ef76fc8529c6" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "5e677518051c43768dbf06243701c817", "version_major": 2, "version_minor": 0 }, "text/plain": [ "toy_face_sdxl.safetensors: 0%| | 0.00/171M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Only UNet\n", "pipe.load_lora_weights(\"CiroN2022/toy-face\", weight_name=\"toy_face_sdxl.safetensors\", adapter_name=\"toy\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "p-7YWoOs02La" }, "outputs": [], "source": [ "from peft import get_peft_model, LoraConfig\n", "\n", "toy_peft_model = get_peft_model(\n", " sdxl_unet,\n", " pipe.unet.peft_config[\"toy\"],\n", " adapter_name=\"toy\"\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 101, "referenced_widgets": [ "c6bdeef396174d51af9eee277752bec7", "685797f8907c47ffab4fe7a81ca22e63", "414f8301f76043758f69bbfb6960072d", "23c0492f021a4b60b1d84b0b82d15378", "550b3ad10fcb422eb66f71ad95988616", "222da37b5af14d60814c65cdd1ea20be", "6a2b5bbd4afe4e0cabe39e95ccf528be", "7ba112dcece64386bdbac6837004891b", "2ffbf400ff1f4cc9b358bb10c6b9d99f", "5405d8a1aa1d411b895b0523fb8e4ce7", "04846af1def142ffad328d434ba228fe" ] }, "id": "a_2n4Odz2a0c", "outputId": "4b3b801b-649f-4c69-b75c-f800ac75c17f" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "c6bdeef396174d51af9eee277752bec7", "version_major": 2, "version_minor": 0 }, "text/plain": [ "README.md: 0%| | 0.00/5.12k [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.google.colaboratory.intrinsic+json": { "type": "string" }, "text/plain": [ "CommitInfo(commit_url='https://huggingface.co/sayakpaul/toy_peft_model-new/commit/85a43b6eae5c1634827e0457a45220da8cf17d75', commit_message='Upload model', commit_description='', oid='85a43b6eae5c1634827e0457a45220da8cf17d75', pr_url=None, pr_revision=None, pr_num=None)" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "original_state_dict = {f\"base_model.model.{k}\": v for k, v in pipe.unet.state_dict().items()}\n", "\n", "toy_peft_model.load_state_dict(original_state_dict, strict=True)\n", "toy_peft_model.push_to_hub(\"toy_peft_model-new\", token=TOKEN)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "z1DWL0X12rxD" }, "outputs": [], "source": [ "pipe.delete_adapters(\"toy\")\n", "sdxl_unet.delete_adapters(\"toy\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 49, "referenced_widgets": [ "d18c97fe685e4be080125ae770526255", "e44c018bc76c48cd8738fee5966767ce", "57b491647f1e49cea7ce34774a963936", "484f694f734b4a92a14ecc4d048db0af", "bcb3ec98d25b4c138c5e8f84c1e937c6", "b0abce2d8a2046dba320e788e33e9d66", "aef81ec1a7e844f883beba8c5754a8af", "c040aa1f65514be28f0ca8ecdd1f69e4", "eec76868d92f45d7a6db2e232a45e0c2", "157c0c1e85ef40fb99e6cfe0e176be38", "99717def9b6b4afe8a411a3bb83320c9" ] }, "id": "9PW-SfwH5L7e", "outputId": "90721ffa-faa5-4628-994a-7b719a4ef02c" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "d18c97fe685e4be080125ae770526255", "version_major": 2, "version_minor": 0 }, "text/plain": [ "pixel-art-xl.safetensors: 0%| | 0.00/171M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "pipe.load_lora_weights(\"nerijs/pixel-art-xl\", weight_name=\"pixel-art-xl.safetensors\", adapter_name=\"pixel\")\n", "pipe.set_adapters(adapter_names=\"pixel\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 101, "referenced_widgets": [ "adb2daf9d62f49f8ab1f0144b717d41e", "bba38c266ceb4f30bb4bc1eaf5e3aa96", "6926c8dd4e4e46d089bb387333691df7", "abf2248c725b4837b5c2babef7f4ff3e", "39e4bdf8d621451f984f6e7302fd6961", "4abac6a83641414499f9b6b1514d1695", "bf6e103b43844f17968eadf223a42acb", "adf53d97af214cceaae895c8abfbd909", "9d74948f7bcb498c9295e41e690a0a8d", "d872c3900b8b4275b2224c9ec5e7d78f", "4fbaa1bd51bf4b1e90337a435604d2bd" ] }, "id": "jHSb-iIf7IEb", "outputId": "29124d4c-b58f-4f0e-c59b-d79b44cb162f" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "adb2daf9d62f49f8ab1f0144b717d41e", "version_major": 2, "version_minor": 0 }, "text/plain": [ "adapter_model.safetensors: 0%| | 0.00/170M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.google.colaboratory.intrinsic+json": { "type": "string" }, "text/plain": [ "CommitInfo(commit_url='https://huggingface.co/sayakpaul/pixel_peft_model-new/commit/02cfa783ac39e0dc8de871298f8d6632ac796b0d', commit_message='Upload model', commit_description='', oid='02cfa783ac39e0dc8de871298f8d6632ac796b0d', pr_url=None, pr_revision=None, pr_num=None)" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pixel_peft_model = get_peft_model(\n", " sdxl_unet,\n", " pipe.unet.peft_config[\"pixel\"],\n", " adapter_name=\"pixel\"\n", ")\n", "\n", "original_state_dict = {f\"base_model.model.{k}\": v for k, v in pipe.unet.state_dict().items()}\n", "pixel_peft_model.load_state_dict(original_state_dict, strict=True)\n", "pixel_peft_model.push_to_hub(\"pixel_peft_model-new\", token=TOKEN)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "yoPzMtyqG2ZO" }, "outputs": [], "source": [ "del pipe, sdxl_unet, toy_peft_model, pixel_peft_model" ] }, { "cell_type": "markdown", "metadata": { "id": "Zis3zKZpGy8w" }, "source": [ "## Weighted adapter inference" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 145, "referenced_widgets": [ "d74b1c667d42472b865ee1cbefc33a60", "1074a274530e46c5b5a3e43653da43d0", "dfde3984062442869bc8091bb94f2c36", "a6f2ce8830734be4b8efe5ca0e14e990", "07da49088e8d480fad03a2e828357872", "5dd09616ba6544c499271054e8d8d2c5", "fdb1fd279a0241429e5721ae2e92d217", "7c6c4dff0a814bc6a7ac677980b45add", "9a9da0d0e3d84a19b5d188c9bd6a83bb", "de31002ed7fc475c915b4a29253108af", "fbf3d268f30344b7864ce691d5bcb1f3", "153a93d930df4ee396e03f1aaa6f04f1", "c8e40d44aeac47a78f6502771f1471b7", "270ddad8d7704f929a28c9fbbdabfa26", "e57d317b3dda43bba13ecd4514f776d3", "22b55b5ee1af4ea7b1acfe511b194cd8", "a48d3ad24e9744f7898d1f2c5a696ea2", "f29673e57d174839a0bde70bfa165715", "13f5160dd981465890ada8a2cef22d5e", "cc06ed7338b74ae6a1c563212aeb9f94", "3a017f1d0ebf4a4aab57ac3eb0788774", "48643ea67f2f4762bcd27de1d4cf0fd2", "5c3d142907404cef8fd624839a166530", "abaecfe7f39a43bb8fbd655d1a3009f4", "d488374da5e74ec3a5973003590a7d69", "5d8ae9661dce4a1aac8d418d84f3a209", "32132ecbe71a4c2c903932513c2a1aa0", "05700817adac4fdda6c95dd00eaeae38", "5a7d87338ddc45df84080e0096a31631", "cb9a536bc56f4d0ebf285e7f73d4730e", "49bb475a11104e9496f2623e3d5caebd", "887c5eae5b154eccb4a1caa3deef6e94", "e87dffe17f1948e9ba794eddb605a908", "2c352a90375443da835eef55b3e63303", "db8fd6b2687c449fa0600d3e87c96999", "17691a346ca5407a99a5e385450c97eb", "0bca833a6aa74ebaa8f69feb738806bf", "6bdb9b0b68c24b84a748c18ed927a8d3", "355efc45ddaf42498d72d6134a28c87b", "8b6464ce614c4aa29ac66ecce29b6cbf", "bc07cdaad5b64fb3b0e1f8c214bba813", "9e2e87c131a140a2a37dfdf483d27ced", "47468a75637d436f849283c295a74ab6", "af5003cf40ae4dfaa0660f247598856e" ] }, "id": "gEqT1vFtG0_e", "outputId": "282ce865-c653-4912-e497-ff825c896ae7" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "d74b1c667d42472b865ee1cbefc33a60", "version_major": 2, "version_minor": 0 }, "text/plain": [ "toy/adapter_config.json: 0%| | 0.00/47.3k [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "153a93d930df4ee396e03f1aaa6f04f1", "version_major": 2, "version_minor": 0 }, "text/plain": [ "adapter_model.safetensors: 0%| | 0.00/170M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "5c3d142907404cef8fd624839a166530", "version_major": 2, "version_minor": 0 }, "text/plain": [ "pixel/adapter_config.json: 0%| | 0.00/47.3k [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "2c352a90375443da835eef55b3e63303", "version_major": 2, "version_minor": 0 }, "text/plain": [ "adapter_model.safetensors: 0%| | 0.00/170M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from peft import PeftModel\n", "\n", "base_unet = UNet2DConditionModel.from_pretrained(\n", " model_id, subfolder=\"unet\", torch_dtype=torch.float16, use_safetensors=True, variant=\"fp16\"\n", ").to(\"cuda\")\n", "\n", "toy_id = \"sayakpaul/toy_peft_model-new\"\n", "model = PeftModel.from_pretrained(base_unet, toy_id, use_safetensors=True, subfolder=\"toy\", adapter_name=\"toy\")\n", "model.load_adapter(\"sayakpaul/pixel_peft_model-new\", use_safetensors=True, subfolder=\"pixel\", adapter_name=\"pixel\")\n", "\n", "# https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraModel.add_weighted_adapter\n", "model.add_weighted_adapter(\n", " adapters=[\"toy\", \"pixel\"],\n", " weights=[0.7, 0.3],\n", " combination_type=\"linear\",\n", " adapter_name=\"toy-pixel\"\n", ")\n", "model.set_adapters(\"toy-pixel\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 186 }, "id": "QStyurhKsP_g", "outputId": "5e3a2627-27a2-4771-e62f-1b81ded2b87e" }, "outputs": [ { "data": { "text/html": [ "
diffusers.models.unets.unet_2d_condition.UNet2DConditionModel
def _wrapped_call_impl(*args, **kwargs)
A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample\n", "shaped output.\n", "\n", "This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented\n", "for all models (such as downloading or saving).\n", "\n", "Parameters:\n", " sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`):\n", " Height and width of input/output sample.\n", " in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample.\n", " out_channels (`int`, *optional*, defaults to 4): Number of channels in the output.\n", " center_input_sample (`bool`, *optional*, defaults to `False`): Whether to center the input sample.\n", " flip_sin_to_cos (`bool`, *optional*, defaults to `False`):\n", " Whether to flip the sin to cos in the time embedding.\n", " freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding.\n", " down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`):\n", " The tuple of downsample blocks to use.\n", " mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`):\n", " Block type for middle of UNet, it can be one of `UNetMidBlock2DCrossAttn`, `UNetMidBlock2D`, or\n", " `UNetMidBlock2DSimpleCrossAttn`. If `None`, the mid block layer is skipped.\n", " up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`):\n", " The tuple of upsample blocks to use.\n", " only_cross_attention(`bool` or `Tuple[bool]`, *optional*, default to `False`):\n", " Whether to include self-attention in the basic transformer blocks, see\n", " [`~models.attention.BasicTransformerBlock`].\n", " block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`):\n", " The tuple of output channels for each block.\n", " layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block.\n", " downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution.\n", " mid_block_scale_factor (`float`, *optional*, defaults to 1.0): The scale factor to use for the mid block.\n", " dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.\n", " act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use.\n", " norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization.\n", " If `None`, normalization and activation layers is skipped in post-processing.\n", " norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization.\n", " cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280):\n", " The dimension of the cross attention features.\n", " transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple]` , *optional*, defaults to 1):\n", " The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for\n", " [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],\n", " [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].\n", " reverse_transformer_layers_per_block : (`Tuple[Tuple]`, *optional*, defaults to None):\n", " The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`], in the upsampling\n", " blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `Tuple[Tuple]` and for\n", " [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],\n", " [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].\n", " encoder_hid_dim (`int`, *optional*, defaults to None):\n", " If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim`\n", " dimension to `cross_attention_dim`.\n", " encoder_hid_dim_type (`str`, *optional*, defaults to `None`):\n", " If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text\n", " embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`.\n", " attention_head_dim (`int`, *optional*, defaults to 8): The dimension of the attention heads.\n", " num_attention_heads (`int`, *optional*):\n", " The number of attention heads. If not defined, defaults to `attention_head_dim`\n", " resnet_time_scale_shift (`str`, *optional*, defaults to `"default"`): Time scale shift config\n", " for ResNet blocks (see [`~models.resnet.ResnetBlock2D`]). Choose from `default` or `scale_shift`.\n", " class_embed_type (`str`, *optional*, defaults to `None`):\n", " The type of class embedding to use which is ultimately summed with the time embeddings. Choose from `None`,\n", " `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`.\n", " addition_embed_type (`str`, *optional*, defaults to `None`):\n", " Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or\n", " "text". "text" will use the `TextTimeEmbedding` layer.\n", " addition_time_embed_dim: (`int`, *optional*, defaults to `None`):\n", " Dimension for the timestep embeddings.\n", " num_class_embeds (`int`, *optional*, defaults to `None`):\n", " Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing\n", " class conditioning with `class_embed_type` equal to `None`.\n", " time_embedding_type (`str`, *optional*, defaults to `positional`):\n", " The type of position embedding to use for timesteps. Choose from `positional` or `fourier`.\n", " time_embedding_dim (`int`, *optional*, defaults to `None`):\n", " An optional override for the dimension of the projected time embedding.\n", " time_embedding_act_fn (`str`, *optional*, defaults to `None`):\n", " Optional activation function to use only once on the time embeddings before they are passed to the rest of\n", " the UNet. Choose from `silu`, `mish`, `gelu`, and `swish`.\n", " timestep_post_act (`str`, *optional*, defaults to `None`):\n", " The second activation function to use in timestep embedding. Choose from `silu`, `mish` and `gelu`.\n", " time_cond_proj_dim (`int`, *optional*, defaults to `None`):\n", " The dimension of `cond_proj` layer in the timestep embedding.\n", " conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer. conv_out_kernel (`int`,\n", " *optional*, default to `3`): The kernel size of `conv_out` layer. projection_class_embeddings_input_dim (`int`,\n", " *optional*): The dimension of the `class_labels` input when\n", " `class_embed_type="projection"`. Required when `class_embed_type="projection"`.\n", " class_embeddings_concat (`bool`, *optional*, defaults to `False`): Whether to concatenate the time\n", " embeddings with the class embeddings.\n", " mid_block_only_cross_attention (`bool`, *optional*, defaults to `None`):\n", " Whether to use cross attention with the mid block when using the `UNetMidBlock2DSimpleCrossAttn`. If\n", " `only_cross_attention` is given as a single boolean and `mid_block_only_cross_attention` is `None`, the\n", " `only_cross_attention` value is used as the value for `mid_block_only_cross_attention`. Default to `False`\n", " otherwise.\n", " \n", "