Upload ltxv-t2v.ipynb

ComfyUI/ltxv-t2v.ipynb

@@ -0,0 +1,204 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%cd /content\n",
+    "!apt install -qqy\n",
+    "!pip install torchsde einops pyaml diffusers transformers peft accelerate aiohttp kornia spandrel sentencepiece ipywidgets termcolor moviepy==1.0.3\n",
+    "\n",
+    "!git clone https://github.com/comfyanonymous/ComfyUI /content/ComfyUI\n",
+    "!git clone https://github.com/ltdrdata/ComfyUI-Manager /content/ComfyUI/custom_nodes/ComfyUI-Manager\n",
+    "!git clone -b dev https://github.com/camenduru/ComfyUI-Fluxpromptenhancer /content/ComfyUI/custom_nodes/ComfyUI-Fluxpromptenhancer\n",
+    "\n",
+    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/FLUX.1-dev/resolve/main/t5xxl_fp16.safetensors -d /content/ComfyUI/models/clip -o t5xxl_fp16.safetensors\n",
+    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Lightricks/LTX-Video/resolve/main/ltx-video-2b-v0.9.safetensors -d /content/ComfyUI/models/checkpoints -o ltx-video-2b-v0.9.safetensors\n",
+    "\n",
+    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/raw/main/config.json -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o config.json\n",
+    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/raw/main/generation_config.json -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o generation_config.json\n",
+    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/resolve/main/model.safetensors -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o model.safetensors\n",
+    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/raw/main/special_tokens_map.json -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o special_tokens_map.json\n",
+    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/resolve/main/spiece.model -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o spiece.model\n",
+    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/raw/main/tokenizer.json -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o tokenizer.json\n",
+    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/raw/main/tokenizer_config.json -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o tokenizer_config.json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%cd /content/ComfyUI\n",
+    "\n",
+    "import os, shutil, json, requests, random, time\n",
+    "from urllib.parse import urlsplit\n",
+    "\n",
+    "import torch\n",
+    "from PIL import Image\n",
+    "from moviepy.editor import ImageSequenceClip\n",
+    "import numpy as np\n",
+    "\n",
+    "from nodes import NODE_CLASS_MAPPINGS, load_custom_node\n",
+    "from comfy_extras import  nodes_images, nodes_lt, nodes_custom_sampler\n",
+    "\n",
+    "load_custom_node(\"/content/ComfyUI/custom_nodes/ComfyUI-Fluxpromptenhancer\")\n",
+    "\n",
+    "FluxPromptEnhance = NODE_CLASS_MAPPINGS[\"FluxPromptEnhance\"]()\n",
+    "CLIPLoader = NODE_CLASS_MAPPINGS[\"CLIPLoader\"]()\n",
+    "CLIPTextEncode = NODE_CLASS_MAPPINGS[\"CLIPTextEncode\"]()\n",
+    "CheckpointLoaderSimple = NODE_CLASS_MAPPINGS[\"CheckpointLoaderSimple\"]()\n",
+    "LTXVConditioning = nodes_lt.NODE_CLASS_MAPPINGS[\"LTXVConditioning\"]()\n",
+    "SamplerCustom = nodes_custom_sampler.NODE_CLASS_MAPPINGS[\"SamplerCustom\"]()\n",
+    "KSamplerSelect = nodes_custom_sampler.NODE_CLASS_MAPPINGS[\"KSamplerSelect\"]()\n",
+    "LTXVScheduler = nodes_lt.NODE_CLASS_MAPPINGS[\"LTXVScheduler\"]()\n",
+    "EmptyLTXVLatentVideo = nodes_lt.NODE_CLASS_MAPPINGS[\"EmptyLTXVLatentVideo\"]()\n",
+    "VAEDecode = NODE_CLASS_MAPPINGS[\"VAEDecode\"]()\n",
+    "SaveAnimatedWEBP = nodes_images.NODE_CLASS_MAPPINGS[\"SaveAnimatedWEBP\"]()\n",
+    "\n",
+    "with torch.inference_mode():\n",
+    "    clip = CLIPLoader.load_clip(\"t5xxl_fp16.safetensors\", type=\"ltxv\")[0]\n",
+    "    model, _, vae = CheckpointLoaderSimple.load_checkpoint(\"ltx-video-2b-v0.9.safetensors\")\n",
+    "\n",
+    "def download_file(url, save_dir, file_name):\n",
+    "    os.makedirs(save_dir, exist_ok=True)\n",
+    "    file_suffix = os.path.splitext(urlsplit(url).path)[1]\n",
+    "    file_name_with_suffix = file_name + file_suffix\n",
+    "    file_path = os.path.join(save_dir, file_name_with_suffix)\n",
+    "    response = requests.get(url)\n",
+    "    response.raise_for_status()\n",
+    "    with open(file_path, 'wb') as file:\n",
+    "        file.write(response.content)\n",
+    "    return file_path\n",
+    "\n",
+    "def webp_to_mp4(input_webp, output_mp4, fps=10):\n",
+    "    with Image.open(input_webp) as img:\n",
+    "        frames = []\n",
+    "        try:\n",
+    "            while True:\n",
+    "                frame = img.copy()\n",
+    "                frames.append(frame)\n",
+    "                img.seek(img.tell() + 1)\n",
+    "        except EOFError:\n",
+    "            pass\n",
+    "    temp_dir = \"temp_frames\"\n",
+    "    os.makedirs(temp_dir, exist_ok=True)\n",
+    "    temp_frame_paths = []\n",
+    "    for i, frame in enumerate(frames):\n",
+    "        frame_path = os.path.join(temp_dir, f\"frame_{i:04d}.png\")\n",
+    "        frame.save(frame_path)\n",
+    "        temp_frame_paths.append(frame_path)\n",
+    "    clip = ImageSequenceClip(temp_frame_paths, fps=fps)\n",
+    "    clip.write_videofile(output_mp4, codec=\"libx264\", fps=fps)\n",
+    "    for path in temp_frame_paths:\n",
+    "        os.remove(path)\n",
+    "    os.rmdir(temp_dir)\n",
+    "\n",
+    "@torch.inference_mode()\n",
+    "def generate(input):\n",
+    "    values = input[\"input\"]\n",
+    "\n",
+    "    positive_prompt = values['positive_prompt']\n",
+    "    negative_prompt = values['negative_prompt']\n",
+    "    frame_rate = values['frame_rate']\n",
+    "    sampler_name = values['sampler_name']\n",
+    "    steps = values['steps']\n",
+    "    max_shift = values['max_shift']\n",
+    "    base_shift = values['base_shift']\n",
+    "    stretch = values['stretch']\n",
+    "    terminal = values['terminal']\n",
+    "    width = values['width']\n",
+    "    height = values['height']\n",
+    "    length = values['length']\n",
+    "    add_noise = values['add_noise']\n",
+    "    noise_seed = values['noise_seed']\n",
+    "    cfg = values['cfg']\n",
+    "    fps = values['fps']\n",
+    "    prompt_enhance = values['prompt_enhance']\n",
+    "\n",
+    "    if noise_seed == 0:\n",
+    "        random.seed(int(time.time()))\n",
+    "        noise_seed = random.randint(0, 18446744073709551615)\n",
+    "    print(noise_seed)\n",
+    "\n",
+    "    if prompt_enhance:\n",
+    "        positive_prompt = FluxPromptEnhance.enhance_prompt(positive_prompt, noise_seed)[0]\n",
+    "        print(positive_prompt)\n",
+    "    conditioning_positive = CLIPTextEncode.encode(clip, positive_prompt)[0]\n",
+    "    conditioning_negative = CLIPTextEncode.encode(clip, negative_prompt)[0]\n",
+    "    positive, negative = LTXVConditioning.append(conditioning_positive, conditioning_negative, frame_rate)\n",
+    "    sampler = KSamplerSelect.get_sampler(sampler_name)[0]\n",
+    "    sigmas = LTXVScheduler.get_sigmas(steps, max_shift, base_shift, stretch, terminal, latent=None)[0]\n",
+    "    latent_image = EmptyLTXVLatentVideo.generate(width, height, length, batch_size=1)[0]\n",
+    "    samples = SamplerCustom.sample(model, add_noise, noise_seed, cfg, positive, negative, sampler, sigmas, latent_image)[0]\n",
+    "    images = VAEDecode.decode(vae, samples)[0].detach()\n",
+    "    video = SaveAnimatedWEBP.save_images(images, fps, filename_prefix=f\"ltx-video-{noise_seed}-tost\", lossless=False, quality=90, method=\"default\")\n",
+    "    source = video['ui']['images'][0]['filename']\n",
+    "    source = f\"/content/ComfyUI/output/{source}\"\n",
+    "    destination = f\"/content/ltx-video-{noise_seed}-tost.webp\"\n",
+    "    shutil.move(source, destination)\n",
+    "    webp_to_mp4(f\"/content/ltx-video-{noise_seed}-tost.webp\", f\"/content/ltx-video-{noise_seed}-tost.mp4\", fps=fps)\n",
+    "    \n",
+    "    result = f\"/content/ltx-video-{noise_seed}-tost.mp4\"\n",
+    "\n",
+    "    return result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "input = { \n",
+    "        \"input\": {\n",
+    "            \"positive_prompt\": \"The camera follows behind a white vintage SUV with a black roof rack as it speeds up a steep dirt road surrounded by pine trees on a steep mountain slope, dust kicks up from it’s tires, the sunlight shines on the SUV as it speeds along the dirt road, casting a warm glow over the scene. The dirt road curves gently into the distance, with no other cars or vehicles in sight. The trees on either side of the road are redwoods, with patches of greenery scattered throughout. The car is seen from the rear following the curve with ease, making it seem as if it is on a rugged drive through the rugged terrain. The dirt road itself is surrounded by steep hills and mountains, with a clear blue sky above with wispy clouds.\",\n",
+    "            \"negative_prompt\": \"low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly\",\n",
+    "            \"frame_rate\": 25,\n",
+    "            \"sampler_name\": \"euler\",\n",
+    "            \"steps\": 30,\n",
+    "            \"max_shift\": 2.05,\n",
+    "            \"base_shift\": 0.95,\n",
+    "            \"stretch\": True,\n",
+    "            \"terminal\": 0.10,\n",
+    "            \"width\": 768,\n",
+    "            \"height\": 512,\n",
+    "            \"length\": 97,\n",
+    "            \"add_noise\": True,\n",
+    "            \"noise_seed\": 0,\n",
+    "            \"cfg\": 3.0,\n",
+    "            \"fps\": 24,\n",
+    "            \"prompt_enhance\": True\n",
+    "    }\n",
+    "}\n",
+    "video = generate(input)\n",
+    "from IPython.display import Video\n",
+    "Video(video, embed=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "ComfyUI0-venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}