| | <!doctype html> |
| | <html> |
| |
|
| | <head> |
| | <meta charset="UTF-8"> |
| | <title>Real-Time Latent Consistency Model ControlNet</title> |
| | <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| | <script |
| | src="https://cdnjs.cloudflare.com/ajax/libs/iframe-resizer/4.3.1/iframeResizer.contentWindow.min.js"></script> |
| | <script src="https://cdn.jsdelivr.net/npm/piexifjs@1.0.6/piexif.min.js"></script> |
| | <script src="https://cdn.tailwindcss.com"></script> |
| | <style type="text/tailwindcss"> |
| | .button { |
| | @apply bg-gray-700 hover:bg-gray-800 text-white font-normal p-2 rounded disabled:bg-gray-300 dark:disabled:bg-gray-700 disabled:cursor-not-allowed dark:disabled:text-black |
| | } |
| | </style> |
| | <script type="module"> |
| | const getValue = (id) => { |
| | const el = document.querySelector(`${id}`) |
| | if (el.type === "checkbox") |
| | return el.checked; |
| | return el.value; |
| | } |
| | const startBtn = document.querySelector("#start"); |
| | const stopBtn = document.querySelector("#stop"); |
| | const videoEl = document.querySelector("#webcam"); |
| | const imageEl = document.querySelector("#player"); |
| | const queueSizeEl = document.querySelector("#queue_size"); |
| | const errorEl = document.querySelector("#error"); |
| | const snapBtn = document.querySelector("#snap"); |
| | const webcamsEl = document.querySelector("#webcams"); |
| | |
| | function LCMLive(webcamVideo, liveImage) { |
| | let websocket; |
| | |
| | async function start() { |
| | return new Promise((resolve, reject) => { |
| | const websocketURL = `${window.location.protocol === "https:" ? "wss" : "ws" |
| | }:${window.location.host}/ws`; |
| | |
| | const socket = new WebSocket(websocketURL); |
| | socket.onopen = () => { |
| | console.log("Connected to websocket"); |
| | }; |
| | socket.onclose = () => { |
| | console.log("Disconnected from websocket"); |
| | stop(); |
| | resolve({ "status": "disconnected" }); |
| | }; |
| | socket.onerror = (err) => { |
| | console.error(err); |
| | reject(err); |
| | }; |
| | socket.onmessage = (event) => { |
| | const data = JSON.parse(event.data); |
| | switch (data.status) { |
| | case "success": |
| | break; |
| | case "start": |
| | const userId = data.userId; |
| | initVideoStream(userId); |
| | break; |
| | case "timeout": |
| | stop(); |
| | resolve({ "status": "timeout" }); |
| | case "error": |
| | stop(); |
| | reject(data.message); |
| | |
| | } |
| | }; |
| | websocket = socket; |
| | }) |
| | } |
| | function switchCamera() { |
| | const constraints = { |
| | audio: false, |
| | video: { width: 1024, height: 1024, deviceId: mediaDevices[webcamsEl.value].deviceId } |
| | }; |
| | navigator.mediaDevices |
| | .getUserMedia(constraints) |
| | .then((mediaStream) => { |
| | webcamVideo.removeEventListener("timeupdate", videoTimeUpdateHandler); |
| | webcamVideo.srcObject = mediaStream; |
| | webcamVideo.onloadedmetadata = () => { |
| | webcamVideo.play(); |
| | webcamVideo.addEventListener("timeupdate", videoTimeUpdateHandler); |
| | }; |
| | }) |
| | .catch((err) => { |
| | console.error(`${err.name}: ${err.message}`); |
| | }); |
| | } |
| | |
| | async function videoTimeUpdateHandler() { |
| | const dimension = getValue("input[name=dimension]:checked"); |
| | const [WIDTH, HEIGHT] = JSON.parse(dimension); |
| | |
| | const canvas = new OffscreenCanvas(WIDTH, HEIGHT); |
| | const videoW = webcamVideo.videoWidth; |
| | const videoH = webcamVideo.videoHeight; |
| | const aspectRatio = WIDTH / HEIGHT; |
| | |
| | const ctx = canvas.getContext("2d"); |
| | ctx.drawImage(webcamVideo, videoW / 2 - videoH * aspectRatio / 2, 0, videoH * aspectRatio, videoH, 0, 0, WIDTH, HEIGHT) |
| | const blob = await canvas.convertToBlob({ type: "image/jpeg", quality: 1 }); |
| | websocket.send(blob); |
| | websocket.send(JSON.stringify({ |
| | "seed": getValue("#seed"), |
| | "prompt": getValue("#prompt"), |
| | "guidance_scale": getValue("#guidance-scale"), |
| | "strength": getValue("#strength"), |
| | "steps": getValue("#steps"), |
| | "lcm_steps": getValue("#lcm_steps"), |
| | "width": WIDTH, |
| | "height": HEIGHT, |
| | "controlnet_scale": getValue("#controlnet_scale"), |
| | "controlnet_start": getValue("#controlnet_start"), |
| | "controlnet_end": getValue("#controlnet_end"), |
| | "canny_low_threshold": getValue("#canny_low_threshold"), |
| | "canny_high_threshold": getValue("#canny_high_threshold"), |
| | "debug_canny": getValue("#debug_canny") |
| | })); |
| | } |
| | let mediaDevices = []; |
| | async function initVideoStream(userId) { |
| | liveImage.src = `/stream/${userId}`; |
| | await navigator.mediaDevices.enumerateDevices() |
| | .then(devices => { |
| | const cameras = devices.filter(device => device.kind === 'videoinput'); |
| | mediaDevices = cameras; |
| | webcamsEl.innerHTML = ""; |
| | cameras.forEach((camera, index) => { |
| | const option = document.createElement("option"); |
| | option.value = index; |
| | option.innerText = camera.label; |
| | webcamsEl.appendChild(option); |
| | option.selected = index === 0; |
| | }); |
| | webcamsEl.addEventListener("change", switchCamera); |
| | }) |
| | .catch(err => { |
| | console.error(err); |
| | }); |
| | const constraints = { |
| | audio: false, |
| | video: { width: 1024, height: 1024, deviceId: mediaDevices[0].deviceId } |
| | }; |
| | navigator.mediaDevices |
| | .getUserMedia(constraints) |
| | .then((mediaStream) => { |
| | webcamVideo.srcObject = mediaStream; |
| | webcamVideo.onloadedmetadata = () => { |
| | webcamVideo.play(); |
| | webcamVideo.addEventListener("timeupdate", videoTimeUpdateHandler); |
| | }; |
| | }) |
| | .catch((err) => { |
| | console.error(`${err.name}: ${err.message}`); |
| | }); |
| | } |
| | |
| | |
| | async function stop() { |
| | websocket.close(); |
| | navigator.mediaDevices.getUserMedia({ video: true }).then((mediaStream) => { |
| | mediaStream.getTracks().forEach((track) => track.stop()); |
| | }); |
| | webcamVideo.removeEventListener("timeupdate", videoTimeUpdateHandler); |
| | webcamsEl.removeEventListener("change", switchCamera); |
| | webcamVideo.srcObject = null; |
| | } |
| | return { |
| | start, |
| | stop |
| | } |
| | } |
| | function toggleMessage(type) { |
| | errorEl.hidden = false; |
| | errorEl.scrollIntoView(); |
| | switch (type) { |
| | case "error": |
| | errorEl.innerText = "To many users are using the same GPU, please try again later."; |
| | errorEl.classList.toggle("bg-red-300", "text-red-900"); |
| | break; |
| | case "success": |
| | errorEl.innerText = "Your session has ended, please start a new one."; |
| | errorEl.classList.toggle("bg-green-300", "text-green-900"); |
| | break; |
| | } |
| | setTimeout(() => { |
| | errorEl.hidden = true; |
| | }, 2000); |
| | } |
| | function snapImage() { |
| | try { |
| | const zeroth = {}; |
| | const exif = {}; |
| | const gps = {}; |
| | zeroth[piexif.ImageIFD.Make] = "LCM Image-to-Image ControNet"; |
| | zeroth[piexif.ImageIFD.ImageDescription] = `prompt: ${getValue("#prompt")} | seed: ${getValue("#seed")} | guidance_scale: ${getValue("#guidance-scale")} | strength: ${getValue("#strength")} | controlnet_start: ${getValue("#controlnet_start")} | controlnet_end: ${getValue("#controlnet_end")} | lcm_steps: ${getValue("#lcm_steps")} | steps: ${getValue("#steps")}`; |
| | zeroth[piexif.ImageIFD.Software] = "https://github.com/radames/Real-Time-Latent-Consistency-Model"; |
| | exif[piexif.ExifIFD.DateTimeOriginal] = new Date().toISOString(); |
| | |
| | const exifObj = { "0th": zeroth, "Exif": exif, "GPS": gps }; |
| | const exifBytes = piexif.dump(exifObj); |
| | |
| | const canvas = document.createElement("canvas"); |
| | canvas.width = imageEl.naturalWidth; |
| | canvas.height = imageEl.naturalHeight; |
| | const ctx = canvas.getContext("2d"); |
| | ctx.drawImage(imageEl, 0, 0); |
| | const dataURL = canvas.toDataURL("image/jpeg"); |
| | const withExif = piexif.insert(exifBytes, dataURL); |
| | |
| | const a = document.createElement("a"); |
| | a.href = withExif; |
| | a.download = `lcm_txt_2_img${Date.now()}.png`; |
| | a.click(); |
| | } catch (err) { |
| | console.log(err); |
| | } |
| | } |
| | |
| | |
| | const lcmLive = LCMLive(videoEl, imageEl); |
| | startBtn.addEventListener("click", async () => { |
| | try { |
| | startBtn.disabled = true; |
| | snapBtn.disabled = false; |
| | const res = await lcmLive.start(); |
| | startBtn.disabled = false; |
| | if (res.status === "timeout") |
| | toggleMessage("success") |
| | } catch (err) { |
| | console.log(err); |
| | toggleMessage("error") |
| | startBtn.disabled = false; |
| | } |
| | }); |
| | stopBtn.addEventListener("click", () => { |
| | lcmLive.stop(); |
| | }); |
| | window.addEventListener("beforeunload", () => { |
| | lcmLive.stop(); |
| | }); |
| | snapBtn.addEventListener("click", snapImage); |
| | setInterval(() => |
| | fetch("/queue_size") |
| | .then((res) => res.json()) |
| | .then((data) => { |
| | queueSizeEl.innerText = data.queue_size; |
| | }) |
| | .catch((err) => { |
| | console.log(err); |
| | }) |
| | , 5000); |
| | </script> |
| | </head> |
| |
|
| | <body class="text-black dark:bg-gray-900 dark:text-white"> |
| | <div class="fixed right-2 top-2 p-4 font-bold text-sm rounded-lg max-w-xs text-center" id="error"> |
| | </div> |
| | <main class="container mx-auto px-4 py-4 max-w-4xl flex flex-col gap-4"> |
| | <article class="text-center max-w-xl mx-auto"> |
| | <h1 class="text-3xl font-bold">Real-Time Latent Consistency Model</h1> |
| | <h2 class="text-2xl font-bold mb-4">ControlNet</h2> |
| | <p class="text-sm"> |
| | This demo showcases |
| | <a href="https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7" target="_blank" |
| | class="text-blue-500 underline hover:no-underline">LCM</a> Image to Image pipeline |
| | using |
| | <a href="https://github.com/huggingface/diffusers/tree/main/examples/community#latent-consistency-pipeline" |
| | target="_blank" class="text-blue-500 underline hover:no-underline">Diffusers</a> with a MJPEG |
| | stream server. |
| | </p> |
| | <p class="text-sm"> |
| | There are <span id="queue_size" class="font-bold">0</span> user(s) sharing the same GPU, affecting |
| | real-time performance. Maximum queue size is 4. <a |
| | href="https://huggingface.co/spaces/radames/Real-Time-Latent-Consistency-Model?duplicate=true" |
| | target="_blank" class="text-blue-500 underline hover:no-underline">Duplicate</a> and run it on your |
| | own GPU. |
| | </p> |
| | </article> |
| | <div> |
| | <h2 class="font-medium">Prompt</h2> |
| | <p class="text-sm text-gray-500"> |
| | Change the prompt to generate different images, accepts <a |
| | href="https://github.com/damian0815/compel/blob/main/doc/syntax.md" target="_blank" |
| | class="text-blue-500 underline hover:no-underline">Compel</a> syntax. |
| | </p> |
| | <div class="flex text-normal px-1 py-1 border border-gray-700 rounded-md items-center"> |
| | <textarea type="text" id="prompt" class="font-light w-full px-3 py-2 mx-1 outline-none dark:text-black" |
| | title="Prompt, this is an example, feel free to modify" |
| | placeholder="Add your prompt here...">Portrait of The Terminator with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5, cinematic, masterpiece</textarea> |
| | </div> |
| | </div> |
| | <div class=""> |
| | <details> |
| | <summary class="font-medium cursor-pointer">Advanced Options</summary> |
| | <div class="grid grid-cols-3 sm:grid-cols-6 items-center gap-3 py-3"> |
| | <label for="webcams" class="text-sm font-medium">Camera Options: </label> |
| | <select id="webcams" class="text-sm border-2 border-gray-500 rounded-md font-light dark:text-black"> |
| | </select> |
| | <div></div> |
| | <label class="text-sm font-medium " for="steps">Inference Steps |
| | </label> |
| | <input type="range" id="steps" name="steps" min="1" max="20" value="4" |
| | oninput="this.nextElementSibling.value = Number(this.value)"> |
| | <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> |
| | 4</output> |
| | |
| | <label class="text-sm font-medium" for="lcm_steps">LCM Inference Steps |
| | </label> |
| | <input type="range" id="lcm_steps" name="lcm_steps" min="2" max="60" value="50" |
| | oninput="this.nextElementSibling.value = Number(this.value)"> |
| | <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> |
| | 50</output> |
| | |
| | <label class="text-sm font-medium" for="guidance-scale">Guidance Scale |
| | </label> |
| | <input type="range" id="guidance-scale" name="guidance-scale" min="0" max="30" step="0.001" |
| | value="8.0" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> |
| | <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> |
| | 8.0</output> |
| | |
| | <label class="text-sm font-medium" for="strength">Strength</label> |
| | <input type="range" id="strength" name="strength" min="0.1" max="1" step="0.001" value="0.50" |
| | oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> |
| | <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> |
| | 0.5</output> |
| | |
| | <label class="text-sm font-medium" for="controlnet_scale">ControlNet Condition Scale</label> |
| | <input type="range" id="controlnet_scale" name="controlnet_scale" min="0.0" max="1" step="0.001" |
| | value="0.80" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> |
| | <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> |
| | 0.8</output> |
| | |
| | <label class="text-sm font-medium" for="controlnet_start">ControlNet Guidance Start</label> |
| | <input type="range" id="controlnet_start" name="controlnet_start" min="0.0" max="1.0" step="0.001" |
| | value="0.0" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> |
| | <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> |
| | 0.0</output> |
| | |
| | <label class="text-sm font-medium" for="controlnet_end">ControlNet Guidance End</label> |
| | <input type="range" id="controlnet_end" name="controlnet_end" min="0.0" max="1.0" step="0.001" |
| | value="1.0" oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> |
| | <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> |
| | 1.0</output> |
| | |
| | <label class="text-sm font-medium" for="canny_low_threshold">Canny Low Threshold</label> |
| | <input type="range" id="canny_low_threshold" name="canny_low_threshold" min="0.0" max="1.0" |
| | step="0.001" value="0.1" |
| | oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> |
| | <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> |
| | 0.1</output> |
| | |
| | <label class="text-sm font-medium" for="canny_high_threshold">Canny High Threshold</label> |
| | <input type="range" id="canny_high_threshold" name="canny_high_threshold" min="0.0" max="1.0" |
| | step="0.001" value="0.2" |
| | oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)"> |
| | <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> |
| | 0.2</output> |
| | |
| | <label class="text-sm font-medium" for="seed">Seed</label> |
| | <input type="number" id="seed" name="seed" value="299792458" |
| | class="font-light border border-gray-700 text-right rounded-md p-2 dark:text-black"> |
| | <button |
| | onclick="document.querySelector('#seed').value = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)" |
| | class="button"> |
| | Rand |
| | </button> |
| | |
| | |
| | <label class="text-sm font-medium" for="dimension">Image Dimensions</label> |
| | <div class="col-span-2 flex gap-2"> |
| | <div class="flex gap-1"> |
| | <input type="radio" id="dimension512" name="dimension" value="[512,512]" checked |
| | class="cursor-pointer"> |
| | <label for="dimension512" class="text-sm cursor-pointer">512x512</label> |
| | </div> |
| | <div class="flex gap-1"> |
| | <input type="radio" id="dimension768" name="dimension" value="[768,768]" |
| | lass="cursor-pointer"> |
| | <label for="dimension768" class="text-sm cursor-pointer">768x768</label> |
| | </div> |
| | </div> |
| | |
| | |
| | <label class="text-sm font-medium" for="debug_canny">Debug Canny</label> |
| | <div class="col-span-2 flex gap-2"> |
| | <input type="checkbox" id="debug_canny" name="debug_canny" class="cursor-pointer"> |
| | <label for="debug_canny" class="text-sm cursor-pointer"></label> |
| | </div> |
| | <div></div> |
| | |
| | </div> |
| | </details> |
| | </div> |
| | <div class="flex gap-3"> |
| | <button id="start" class="button"> |
| | Start |
| | </button> |
| | <button id="stop" class="button"> |
| | Stop |
| | </button> |
| | <button id="snap" disabled class="button ml-auto"> |
| | Snapshot |
| | </button> |
| | </div> |
| | <div class="relative rounded-lg border border-slate-300 overflow-hidden"> |
| | <img id="player" class="w-full aspect-square rounded-lg" |
| | src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="> |
| | <div class="absolute top-0 left-0 w-1/4 aspect-square"> |
| | <video id="webcam" class="w-full aspect-square relative z-10 object-cover" playsinline autoplay muted |
| | loop></video> |
| | <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 448" width="100" |
| | class="w-full p-4 absolute top-0 opacity-20 z-0"> |
| | <path fill="currentColor" |
| | d="M224 256a128 128 0 1 0 0-256 128 128 0 1 0 0 256zm-45.7 48A178.3 178.3 0 0 0 0 482.3 29.7 29.7 0 0 0 29.7 512h388.6a29.7 29.7 0 0 0 29.7-29.7c0-98.5-79.8-178.3-178.3-178.3h-91.4z" /> |
| | </svg> |
| | </div> |
| | </div> |
| | </main> |
| | </body> |
| |
|
| | </html> |
