Added: Generate image to video

app.py

@@ -10,13 +10,16 @@ import uuid
 from dotenv import load_dotenv
 from flask import Flask, request, jsonify, render_template, send_file
 from PIL import Image
+from werkzeug.utils import secure_filename
 
-# Load environment variables from the .env file.
+# Load environment variables from the .env file
 load_dotenv()
 
 # Initialize Flask app
 app = Flask(__name__)
 
+ALLOWED_EXTENSIONS = {'jpg', 'jpeg', 'png', 'webp'}  # Define supported image types
+
 # Set server and websocket addresses from environment variables
 server_address = os.getenv("SERVER_ADDRESS")
 ws_address = os.getenv("WS_ADDRESS")
@@ -24,10 +27,35 @@ ws_address = os.getenv("WS_ADDRESS")
 # Generate a unique client ID
 client_id = str(uuid.uuid4())
 
+def allowed_file(filename):
+    """Check if the uploaded file has an allowed extension."""
+    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+
+def save_base64_image(b64_string):
+    """Decode a base64 string and save it as an image."""
+    header, encoded = b64_string.split(',', 1)  # Handle data URI schemes if provided
+    image_data = base64.b64decode(encoded)
+
+    # Determine image extension from data URI or use a default one
+    ext = header.split('/')[1].split(';')[0] if '/' in header else 'png'
+    image_path = f"/tmp/{uuid.uuid4()}.{ext}"
+
+    with open(image_path, 'wb') as f:
+        f.write(image_data)
+    return image_path
+
 def make_request(url, data=None, headers=None):
     req = urllib.request.Request(url, data=data, headers=headers)
-    with urllib.request.urlopen(req) as response:
-        return json.loads(response.read())
+    try:
+        with urllib.request.urlopen(req) as response:
+            response_body = response.read().decode()  # Decode the response
+            # print(response_body)
+            return json.loads(response_body)  # Convert to JSON if valid
+    except urllib.error.HTTPError as e:
+        print(f"HTTPError: {e.code}, {e.reason}")
+        print(e.read().decode())  # Print detailed error response
+    except urllib.error.URLError as e:
+        print(f"URLError: {e.reason}")
 
 def queue_prompt(prompt, token):
     payload = {"prompt": prompt, "client_id": client_id}
@@ -82,6 +110,12 @@ def get_images(ws, prompt, token):
 
     return output_images
 
+def fetch_video(video_data, token):
+    video_url = f"{server_address}/download?file={video_data['filename']}"
+    req = urllib.request.Request(video_url)
+    req.add_header("Authorization", f"Bearer {token}")
+    return urllib.request.urlopen(req).read()
+
 # Default route for home welcome
 @app.route('/')
 def home():
@@ -124,14 +158,21 @@ def generate_image():
     # seednum = random.randint(1, 9999999999999)
     # prompt["3"]["inputs"]["seed"] = seednum
 
-    # For model Flux1.dev
+                #######################
+                # For model Flux1.dev #
+                #######################
 
     # Generate a random 15-digit seed as an integer
     seednum = random.randint(100000000000000, 999999999999999)
     prompt["31"]["inputs"]["seed"] = seednum
 
     ws = websocket.WebSocket()
-    ws.connect(f"{ws_address}?clientId={client_id}&token={token}")
+
+    try:
+        ws.connect(f"{ws_address}?clientId={client_id}&token={token}")
+    except websocket.WebSocketException as e:
+        return jsonify({'error': f'WebSocket connection failed: {str(e)}'}), 500
+
     images = get_images(ws, prompt, token)
     ws.close()
 
@@ -151,5 +192,77 @@ def generate_image():
 def get_image_file(filename):
     return send_file(filename, mimetype='image/png')
 
+@app.route('/image_to_video', methods=['POST'])
+def image_to_video():
+    data = request.json
+
+    # Extract token from headers
+    token = request.headers.get('Authorization')
+    if not token or not token.startswith("Bearer "):
+        return jsonify({'error': 'Invalid or missing token'}), 400
+    token = base64.b64decode(token.split(" ")[1]).decode("utf-8")
+
+    # Extract text prompt
+    text_prompt = data.get('text_prompt')
+    if not text_prompt:
+        return jsonify({'error': 'Text prompt is required'}), 400
+
+    # Handle uploaded image or base64-encoded image
+    image_file = request.files.get('image')
+    base64_image = data.get('base64_image')
+
+    if image_file:
+        # Validate and save uploaded image
+        if not allowed_file(image_file.filename):
+            return jsonify({'error': 'Unsupported image format'}), 400
+        filename = secure_filename(image_file.filename)
+        image_path = f"/tmp/{uuid.uuid4()}_{filename}"
+        image_file.save(image_path)
+
+    elif base64_image:
+        # Save base64-encoded image
+        try:
+            image_path = save_base64_image(base64_image)
+        except Exception as e:
+            return jsonify({'error': f'Invalid base64 image data: {str(e)}'}), 400
+    else:
+        return jsonify({'error': 'Image is required (either file or base64)'}), 400
+
+    # Get the path to the workflow configuration file
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    file_path = os.path.join(current_dir, 'workflows/cogvideox_image_to_video_workflow_api.json')
+
+    # Load and modify workflow
+    with open(file_path, 'r', encoding='utf-8') as file:
+        workflow = json.load(file)
+    workflow["30"]["inputs"]["prompt"] = text_prompt  # Text prompt
+    workflow["36"]["inputs"]["upload"] = image_path   # Image path
+    workflow["31"]["inputs"]["prompt"] = "Low quality, watermark, strange motion"  # Negative prompt
+
+    seed = random.randint(1e14, 9e14)
+    workflow["57"]["inputs"]["seed"] = seed  # Set reproducibility seed
+
+    # WebSocket connection to trigger workflow
+    ws = websocket.WebSocket()
+    ws.connect(f"{ws_address}?clientId={client_id}&token={token}")
+    ws.send(json.dumps({"workflow": workflow}))  # Send the modified workflow
+
+    # Receive video processing result
+    while True:
+        out = ws.recv()
+        message = json.loads(out)
+        if message.get('type') == 'completed':
+            video_data = message['data']
+            break
+
+    # Fetch and return the generated video
+    video_content = fetch_video(video_data, token)
+    return send_file(
+        io.BytesIO(video_content),
+        mimetype='video/mp4',
+        as_attachment=True,
+        download_name='generated_video.mp4'
+    )
+
 if __name__ == '__main__':
     app.run(host='0.0.0.0', port=7860)  # Removed 'debug=True'

workflows/cogvideox_image_to_video_workflow_api.json

@@ -0,0 +1,176 @@
+{
+  "1": {
+    "inputs": {
+      "model": "THUDM/CogVideoX-5b-I2V",
+      "precision": "bf16",
+      "fp8_transformer": "disabled",
+      "compile": "disabled",
+      "enable_sequential_cpu_offload": false
+    },
+    "class_type": "DownloadAndLoadCogVideoModel",
+    "_meta": {
+      "title": "(Down)load CogVideo Model"
+    }
+  },
+  "20": {
+    "inputs": {
+      "clip_name": "t5\\google_t5-v1_1-xxl_encoderonly-fp8_e4m3fn.safetensors",
+      "type": "sd3"
+    },
+    "class_type": "CLIPLoader",
+    "_meta": {
+      "title": "Load CLIP"
+    }
+  },
+  "30": {
+    "inputs": {
+      "prompt": "Darth Vader is a vampire and blood is dripping from his steel fangs. The camera is slowly rotating around him. The background is filled with smoke and a strong light.",
+      "strength": 1,
+      "force_offload": true,
+      "clip": [
+        "20",
+        0
+      ]
+    },
+    "class_type": "CogVideoTextEncode",
+    "_meta": {
+      "title": "CogVideo TextEncode"
+    }
+  },
+  "31": {
+    "inputs": {
+      "prompt": "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. ",
+      "strength": 1,
+      "force_offload": true,
+      "clip": [
+        "20",
+        0
+      ]
+    },
+    "class_type": "CogVideoTextEncode",
+    "_meta": {
+      "title": "CogVideo TextEncode"
+    }
+  },
+  "36": {
+    "inputs": {
+      "image": "Webimage-1-720x480.jpg",
+      "upload": "image"
+    },
+    "class_type": "LoadImage",
+    "_meta": {
+      "title": "Load Image"
+    }
+  },
+  "37": {
+    "inputs": {
+      "width": 720,
+      "height": 480,
+      "upscale_method": "lanczos",
+      "keep_proportion": false,
+      "divisible_by": 16,
+      "crop": "disabled",
+      "image": [
+        "36",
+        0
+      ]
+    },
+    "class_type": "ImageResizeKJ",
+    "_meta": {
+      "title": "Resize Image"
+    }
+  },
+  "44": {
+    "inputs": {
+      "frame_rate": 24,
+      "loop_count": 0,
+      "filename_prefix": "CogVideoX-I2V",
+      "format": "video/h264-mp4",
+      "pix_fmt": "yuv420p",
+      "crf": 19,
+      "save_metadata": true,
+      "pingpong": false,
+      "save_output": true,
+      "images": [
+        "56",
+        0
+      ]
+    },
+    "class_type": "VHS_VideoCombine",
+    "_meta": {
+      "title": "Video Combine 🎥🅥🅗🅢"
+    }
+  },
+  "56": {
+    "inputs": {
+      "enable_vae_tiling": false,
+      "tile_sample_min_height": 96,
+      "tile_sample_min_width": 96,
+      "tile_overlap_factor_height": 0.083,
+      "tile_overlap_factor_width": 0.083,
+      "auto_tile_size": true,
+      "pipeline": [
+        "57",
+        0
+      ],
+      "samples": [
+        "57",
+        1
+      ]
+    },
+    "class_type": "CogVideoDecode",
+    "_meta": {
+      "title": "CogVideo Decode"
+    }
+  },
+  "57": {
+    "inputs": {
+      "height": 480,
+      "width": 720,
+      "num_frames": 49,
+      "steps": 50,
+      "cfg": 6,
+      "seed": 65334758276105,
+      "scheduler": "DPM",
+      "denoise_strength": 16,
+      "pipeline": [
+        "1",
+        0
+      ],
+      "positive": [
+        "30",
+        0
+      ],
+      "negative": [
+        "31",
+        0
+      ],
+      "image_cond_latents": [
+        "58",
+        0
+      ]
+    },
+    "class_type": "CogVideoSampler",
+    "_meta": {
+      "title": "CogVideo Sampler"
+    }
+  },
+  "58": {
+    "inputs": {
+      "chunk_size": 16,
+      "enable_tiling": true,
+      "pipeline": [
+        "1",
+        0
+      ],
+      "image": [
+        "37",
+        0
+      ]
+    },
+    "class_type": "CogVideoImageEncode",
+    "_meta": {
+      "title": "CogVideo ImageEncode"
+    }
+  }
+}

workflows/deliberate_v6_workflow_api.json

@@ -0,0 +1,107 @@
+{
+  "3": {
+    "inputs": {
+      "seed": 156680208700286,
+      "steps": 20,
+      "cfg": 8,
+      "sampler_name": "euler",
+      "scheduler": "normal",
+      "denoise": 1,
+      "model": [
+        "4",
+        0
+      ],
+      "positive": [
+        "6",
+        0
+      ],
+      "negative": [
+        "7",
+        0
+      ],
+      "latent_image": [
+        "5",
+        0
+      ]
+    },
+    "class_type": "KSampler",
+    "_meta": {
+      "title": "KSampler"
+    }
+  },
+  "4": {
+    "inputs": {
+      "ckpt_name": "Deliberate_v6.safetensors"
+    },
+    "class_type": "CheckpointLoaderSimple",
+    "_meta": {
+      "title": "Load Checkpoint"
+    }
+  },
+  "5": {
+    "inputs": {
+      "width": 512,
+      "height": 512,
+      "batch_size": 1
+    },
+    "class_type": "EmptyLatentImage",
+    "_meta": {
+      "title": "Empty Latent Image"
+    }
+  },
+  "6": {
+    "inputs": {
+      "text": "beautiful scenery nature glass bottle landscape, , purple galaxy bottle,",
+      "clip": [
+        "4",
+        1
+      ]
+    },
+    "class_type": "CLIPTextEncode",
+    "_meta": {
+      "title": "CLIP Text Encode (Prompt)"
+    }
+  },
+  "7": {
+    "inputs": {
+      "text": "text, watermark",
+      "clip": [
+        "4",
+        1
+      ]
+    },
+    "class_type": "CLIPTextEncode",
+    "_meta": {
+      "title": "CLIP Text Encode (Prompt)"
+    }
+  },
+  "8": {
+    "inputs": {
+      "samples": [
+        "3",
+        0
+      ],
+      "vae": [
+        "4",
+        2
+      ]
+    },
+    "class_type": "VAEDecode",
+    "_meta": {
+      "title": "VAE Decode"
+    }
+  },
+  "9": {
+    "inputs": {
+      "filename_prefix": "ComfyUI",
+      "images": [
+        "8",
+        0
+      ]
+    },
+    "class_type": "SaveImage",
+    "_meta": {
+      "title": "Save Image"
+    }
+  }
+}