{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "view-in-github"
},
"source": [
""
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "REl-Wc9qXMIm"
},
"source": [
"# Illusion Diffusion\n",
"By [Matt Tancik](https://www.matthewtancik.com/)\n",
"\n",
"Based on fast.ai [diffusion notebooks](https://github.com/fastai/diffusion-nbs).\n",
"\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 331,
"referenced_widgets": [
"ec93e6f38614427dbcfce3f90080b704",
"250a79dff54c49208331e56446788a34",
"4a679f608885425a8c3fb68e8bf0e59e",
"384806faeaa64d0e91ede64bd693734c",
"217f74737ced4a0bb1b0b0f7b32212df",
"02e9ad5c65a64a65b6ffa1cc9db2bc74",
"6f6d302f6b3642688dcd856861d17f00",
"7dfc93de0bab415aa60d342041d9045c",
"388a780f4205497e94821021b533bafc",
"1a4c895af384431ab79864f569a42f0a",
"a527fa7421f34df79e417d58ac6c01bc",
"e0f93ff7eb0a430890b69f1c412d0330",
"6e9e72df17b543d08c26eb932f130516",
"90ce8ae68b774ca9b98d6c8446e5bbe7",
"7b8676d33e59468a897495b19a9ca0b6",
"cd4c97a81b614a3cbbee280e2df12ebd",
"e6ebf8174bb84f2e9d61f54e4095a355"
]
},
"id": "_AqBsIA5XRQ6",
"outputId": "562532b6-48eb-46a8-f494-5c39c89c0261"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Token is valid.\n",
"Your token has been saved in your configured git credential helpers (store).\n",
"Your token has been saved to /root/.cache/huggingface/token\n",
"Login successful\n"
]
}
],
"source": [
"# @title #Install requirements\n",
"!pip install -q --upgrade transformers diffusers ftfy mediapy\n",
"\n",
"from base64 import b64encode\n",
"\n",
"import torch\n",
"from diffusers import AutoencoderKL, LMSDiscreteScheduler, UNet2DConditionModel\n",
"from huggingface_hub import notebook_login\n",
"\n",
"import numpy as np\n",
"import mediapy as mp\n",
"import cv2\n",
"\n",
"from pathlib import Path\n",
"from torch import autocast\n",
"from tqdm.auto import tqdm\n",
"from transformers import CLIPTextModel, CLIPTokenizer, logging\n",
"\n",
"torch.manual_seed(1)\n",
"if not (Path.home()/'.huggingface'/'token').exists(): notebook_login()\n",
"\n",
"# Supress some unnecessary warnings when loading the CLIPTextModel\n",
"logging.set_verbosity_error()\n",
"\n",
"# Set device\n",
"torch_device = \"cuda\" if torch.cuda.is_available() else \"cpu\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 536,
"referenced_widgets": [
"556705183bd543d3901536cec18ded0d",
"fc44fc81699e436b911fbdf15ea9f3f0",
"70a47ab819084ef8bb2a02436e41d947",
"c9a5967a8619496d846bea386bbb839a",
"d8931b9edd064f549202ab04e4dd3e0c",
"81a80b12037a4fc6bfe4200ed386c38a",
"e495eea5d6a845a5aefe815e3e037786",
"a61f0edcb520432aad103469576c299f",
"7785e2c6aac6447da46cdfd84e330da1",
"12c2e7c2106f4cc1a32adfecec77f172",
"e47209c2a56742088dc60e16d9b64645",
"29b35e3c14354dd395e1d134dda94bf6",
"76e34dc11cce47dab884a1ce3e3c8c10",
"89130143468241b6bdd714039b10e785",
"3e6ecc309ae0481c8aeac6f3cc976b3b",
"010c1672ef604e0091968835b5b4e195",
"24fa47f61cc64fe5b76661861c3e72bb",
"8762daaf87de44978bf813c8e1d69e57",
"4bc3cd29ec064adf9817fb6aec7b2b33",
"f68b9f95d6754df6b3fd832540731967",
"2f2753e8cd0148f1a06c636f97cf06a1",
"4b62e52a52894b31acae2dac9d8578eb",
"05bd3fe20a244990aaebab55cf1cf4b7",
"5f81a511c52047348f704ab57088ea18",
"30a399ab9b7941c9ae86e73ff7ea36be",
"81b324b7a8b54c21b8ff4e4ca8ac4096",
"af3c0a09bcb0410881f85369b79ba827",
"5859bd6581f0492abb99a486c5a5d7ef",
"a0ce2a43853e46e6a1d6e432d86f1041",
"14d2a1984fcf498cbff4f3b7c8877edc",
"6ffacfab3e8948298b02744483609854",
"4f1fb4679edb4406b606aadb3b952f09",
"3c1e406f20ed4fe8a013d26d9cbb8956",
"c113e7e8658e48e6a1083868c8615b36",
"7f31abb867184f2f8c0beeb6552c4068",
"dabe54ec9f8546d091f27f15f1215228",
"adc10be2e26f49559d8b522fff3a10d0",
"fd864062415a44c88d71c3a5eab2846c",
"f513a974e5b849c4847d487b3736088f",
"b236d635824f419687e70b4a60b4f2cf",
"94a919a1c15440eb8e8c3b6493374531",
"3144b58b15284fb1bcb5a2a282a8b5f5",
"3a8795a75d974b20bf4f930746bdb605",
"10114f9dd71f48f3bdce17f695a357db",
"37d562a89bfb46a084d4d523d18cd5c0",
"61ab30a402344190aab109460ff71c46",
"df4e7b3c31ad4da4b0bcf911234221e0",
"fe71ebdd5a044e309dc9184e4fd6c01f",
"06e6d0a8a2d14e2ab732937b39747dcd",
"1edc40c25503463b8573acf9f9d30a21",
"c3d39fd215e041d7b98bda4f7612d971",
"63954b09598c4d0bbb9d55eb452f78eb",
"3db80d32aa77436c8319e576606bd71f",
"352989d69e9b4232879186b2294b93f9",
"19f131353b6a4737bf4d5261b0a56f40",
"0eac566e0d94431ca042085314b205d3",
"de478ed2a1cd4f3c98c7d41b52cc959b",
"989cb530129f43ce8490db8607f434ff",
"8b77cff427034fb297c380416c35df50",
"dcc955db0d1b4428ace877c94a265b4f",
"f55248d5a94e4ff7a97545a1f321293e",
"82053fa25f6947118278ccfb9f6d7897",
"aedca04449b24fdf8d3ea9d760685466",
"6dc6f5d0975b4581a0edd9581dfe3902",
"9e9753f67221465898de67ec6e65fa48",
"ab00e175dffa440d898392a5d0a33236",
"1ca8cf416d724456bc26847f6209d6b2",
"6d951ea99e6b4b91a3f5b0f501391b38",
"bb795e7910d84630ae08809ea2a3d2ca",
"34c99a4f913c47009496f3994ac1666d",
"6fc6069c71e44907bb26d6f29b1f79a0",
"1964f2e00b49405b8ad2e629afcd12ad",
"2f92711cb4fb475eb37dfa31d67960d4",
"c2017e08311d457981b490e1b95a50e2",
"790c6b3797fd48afb8ff3ec09a4887aa",
"af8194897edf471aa4dd33d43ba2af60",
"8b34078941d840c4bf2cc44b11d464a2",
"21c48803d550496b99761ce7c6daff02",
"e245379865d4486b83f91eea4a77e40f",
"a634c7ac77a7458e86b30dde312a4952",
"93cc033873054b91bd61fbc4e86323cf",
"a8582419bde245b5b999cc36984f6bec",
"2d49fd55b6444aafa6bdc07fa82ea9a0",
"b3eeab43babe4df890e15b3277f650b7",
"a42cc23ad7b840fc9d1bd26d49339c83",
"d6995adc179d4d78b94bd580b9695362",
"752e239202ad4422b25207101f961d02",
"f4896cac5f0f4048b1bc4a830419405c",
"7d8b4fb1b8004357b5bb2e16d20bc6e9",
"d247ba6381c14d318210c305c976e93b",
"014d20bc05ab4edb9ec2f0f6f2e21aa0",
"d382862ace684422bcbabc1381186dad",
"26320b6da3ea47fdbe9cc11c680501d1",
"fd805e4314da49b5a075fce39c83671c",
"77e2fb3b83314546af03522ad0e16609",
"b3092db7ae5b4c3aa258164dc8bacbbc",
"2ac393b5a28f46dd95d7251bbb1e1095",
"208d2a8ee50d4555bd5c02bffaba4be0",
"f5d5c3d0994a410e86f8678e42c90a94",
"8534e10856cc415a9f6f576f28fbcb99",
"568cf04404d74590b5b18f75c2ede23c",
"508bbbcb38dc43fcba9f0ee3dc7fadcd",
"bf667f0d02d34e32a0ee51d476b9f1f1",
"49635eb6bc1a4618a2696526494c52a6",
"b071e36b1b6442fea5fb0c5b5c8ea770",
"924bf4d2b02c40b59060c3a891fbb6dd",
"1bd7018f64414d2b811ea51dc1155ef7",
"c91af1eecb9e41bc8e67284eaddad6fb",
"df0fa70d1868473dac5b8455dd6444cd",
"7d7184c109194db4a027d800b4dd872b"
]
},
"id": "g8v90HHmXuHN",
"outputId": "513abf68-309c-401f-c672-542b5ed8a6b9"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Cannot initialize model with low cpu memory usage because `accelerate` was not found in the environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install `accelerate` for faster and less memory-intense model loading. You can do so with: \n",
"```\n",
"pip install accelerate\n",
"```\n",
".\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "556705183bd543d3901536cec18ded0d",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading (…)_pytorch_model.bin\";: 0%| | 0.00/335M [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "29b35e3c14354dd395e1d134dda94bf6",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading (…)main/vae/config.json: 0%| | 0.00/547 [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "05bd3fe20a244990aaebab55cf1cf4b7",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading (…)olve/main/vocab.json: 0%| | 0.00/961k [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c113e7e8658e48e6a1083868c8615b36",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading (…)olve/main/merges.txt: 0%| | 0.00/525k [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "37d562a89bfb46a084d4d523d18cd5c0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading (…)cial_tokens_map.json: 0%| | 0.00/389 [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0eac566e0d94431ca042085314b205d3",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading (…)okenizer_config.json: 0%| | 0.00/905 [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1ca8cf416d724456bc26847f6209d6b2",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading (…)lve/main/config.json: 0%| | 0.00/4.52k [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "21c48803d550496b99761ce7c6daff02",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading (…)\"pytorch_model.bin\";: 0%| | 0.00/1.71G [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Cannot initialize model with low cpu memory usage because `accelerate` was not found in the environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install `accelerate` for faster and less memory-intense model loading. You can do so with: \n",
"```\n",
"pip install accelerate\n",
"```\n",
".\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7d8b4fb1b8004357b5bb2e16d20bc6e9",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading (…)_pytorch_model.bin\";: 0%| | 0.00/3.44G [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "8534e10856cc415a9f6f576f28fbcb99",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading (…)ain/unet/config.json: 0%| | 0.00/743 [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# @title #Load Models\n",
"# Load the autoencoder model which will be used to decode the latents into image space. \n",
"vae = AutoencoderKL.from_pretrained(\"runwayml/stable-diffusion-v1-5\", subfolder=\"vae\", torch_dtype=torch.float16)\n",
"\n",
"# Load the tokenizer and text encoder to tokenize and encode the text. \n",
"tokenizer = CLIPTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
"text_encoder = CLIPTextModel.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
"\n",
"# The UNet model for generating the latents.\n",
"unet = UNet2DConditionModel.from_pretrained(\"runwayml/stable-diffusion-v1-5\", subfolder=\"unet\", torch_dtype=torch.float16)\n",
"\n",
"# The noise scheduler\n",
"scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule=\"scaled_linear\", num_train_timesteps=1000)\n",
"\n",
"# To the GPU we go!\n",
"vae = vae.to(torch_device)\n",
"text_encoder = text_encoder.to(torch_device)\n",
"unet = unet.to(torch_device)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 585,
"referenced_widgets": [
"142f18e2740846d59fdca81a1f4ae1d4",
"b49657637b56444f8a11c9bc2c27cd2a",
"29453840d67f46c1b2e9078981ddec72",
"acfd7314064542a1aa15cc10288b321e",
"984a748767a245d7bb1c3f009b08837a",
"b0ef08db21214be6a0096af130df5438",
"df2116135e054e1d8209c36b34273cf3",
"a9e6c54b628e48ee8606337532c93a45",
"f906b58f538e4404babc423f1eab99d8",
"d0b77d45c6724d69a25f1443fa3e31b1",
"c59a9ff0010d4ac39ca3abbeb48d0ca3"
]
},
"id": "UwfYZde5Yixp",
"outputId": "1c396463-5ff6-43b9-8a6e-fb4647ba56fd"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "142f18e2740846d59fdca81a1f4ae1d4",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"