Added: Generate image to video

app.py

@@ -6,6 +6,7 @@ import random
 import urllib.request
 import urllib.parse
 import websocket
+import requests
 import uuid
 from dotenv import load_dotenv
 from flask import Flask, request, jsonify, render_template, send_file
@@ -27,44 +28,8 @@ ws_address = os.getenv("WS_ADDRESS")
 # Generate a unique client ID
 client_id = str(uuid.uuid4())
 
-def allowed_file(filename):
-    """Check if the uploaded file has an allowed extension."""
-    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
-
-def save_base64_image(b64_string):
-    """Decode a base64 string and save it as an image."""
-    header, encoded = b64_string.split(',', 1)  # Handle data URI schemes if provided
-    image_data = base64.b64decode(encoded)
-
-    # Determine image extension from data URI or use a default one
-    ext = header.split('/')[1].split(';')[0] if '/' in header else 'png'
-    image_path = f"/tmp/{uuid.uuid4()}.{ext}"
 
-    with open(image_path, 'wb') as f:
-        f.write(image_data)
-    return image_path
-
-def make_request(url, data=None, headers=None):
-    req = urllib.request.Request(url, data=data, headers=headers)
-    try:
-        with urllib.request.urlopen(req) as response:
-            response_body = response.read().decode()  # Decode the response
-            # print(response_body)
-            return json.loads(response_body)  # Convert to JSON if valid
-    except urllib.error.HTTPError as e:
-        print(f"HTTPError: {e.code}, {e.reason}")
-        print(e.read().decode())  # Print detailed error response
-    except urllib.error.URLError as e:
-        print(f"URLError: {e.reason}")
 
-def queue_prompt(prompt, token):
-    payload = {"prompt": prompt, "client_id": client_id}
-    data = json.dumps(payload).encode('utf-8')
-    headers = {
-        'Authorization': f'Bearer {token}',
-        'Content-Type': 'application/json'
-    }
-    return make_request(f"{server_address}/prompt", data=data, headers=headers)
 
 def get_image(filename, subfolder, image_type, token):
     url_values = {'filename': filename, 'subfolder': subfolder, 'type': image_type}
@@ -78,15 +43,9 @@ def get_image(filename, subfolder, image_type, token):
         print(e.read())
         raise
 
-def get_history(prompt_id, token):
-    headers = {
-        'Authorization': f'Bearer {token}',
-        'Content-Type': 'application/json'
-    }
-    return make_request(f"{server_address}/history/{prompt_id}", headers=headers)
 
-def get_images(ws, prompt, token):
-    prompt_id = queue_prompt(prompt, token)['prompt_id']
+def get_images(ws, workflow, token):
+    prompt_id = queue_prompt(workflow, token)['prompt_id']
     output_images = {}
 
     while True:
@@ -110,6 +69,7 @@ def get_images(ws, prompt, token):
 
     return output_images
 
+
 def fetch_video(video_data, token):
     video_url = f"{server_address}/download?file={video_data['filename']}"
     req = urllib.request.Request(video_url)
@@ -151,12 +111,12 @@ def generate_image():
     with open(file_path, 'r', encoding='utf-8') as file:
         workflow_jsondata = file.read()
 
-    prompt = json.loads(workflow_jsondata)
-    prompt["6"]["inputs"]["text"] = text_prompt
+    workflow = json.loads(workflow_jsondata)
+    workflow["6"]["inputs"]["text"] = text_prompt
 
-    # prompt["7"]["inputs"]["text"] = "text, watermark, low quality, extra hands, extra legs."
+    # workflow["7"]["inputs"]["text"] = "text, watermark, low quality, extra hands, extra legs."
     # seednum = random.randint(1, 9999999999999)
-    # prompt["3"]["inputs"]["seed"] = seednum
+    # workflow["3"]["inputs"]["seed"] = seednum
 
                 #######################
                 # For model Flux1.dev #
@@ -164,7 +124,7 @@ def generate_image():
 
     # Generate a random 15-digit seed as an integer
     seednum = random.randint(100000000000000, 999999999999999)
-    prompt["31"]["inputs"]["seed"] = seednum
+    workflow["31"]["inputs"]["seed"] = seednum
 
     ws = websocket.WebSocket()
 
@@ -174,7 +134,7 @@ def generate_image():
     except websocket.WebSocketException as e:
         return jsonify({'error': f'WebSocket connection failed: {str(e)}'}), 500
 
-    images = get_images(ws, prompt, token)
+    images = get_images(ws, workflow, token)
     ws.close()
 
     output_images_base64 = []
@@ -194,88 +154,200 @@ def generate_image():
 def get_image_file(filename):
     return send_file(filename, mimetype='image/png')
 
-# Generate image to video route
-@app.route('/image_to_video', methods=['POST'])
-def image_to_video():
 
-    # return "Route works!"
+def allowed_file(filename):
+    """Check if the uploaded file has an allowed extension."""
+    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+
+def save_base64_image(b64_string):
+    """Decode a base64 string and save it as an image."""
+    try:
+        # Handle Data URI scheme if present
+        if ',' in b64_string:
+            header, encoded = b64_string.split(',', 1)
+            ext = header.split('/')[1].split(';')[0] if '/' in header else 'png'
+        else:
+            encoded = b64_string
+            ext = 'png'
+
+        # Decode the image data
+        image_data = base64.b64decode(encoded)
+
+        # Generate a unique path for the image
+        image_path = f"/tmp/{uuid.uuid4()}.{ext}"
 
+        # Ensure directory exists
+        os.makedirs('/tmp/', exist_ok=True)
+
+        # Save the image
+        with open(image_path, 'wb') as f:
+            f.write(image_data)
+
+        print(f"Image saved at: {image_path}")
+        return (f"https://gosign-de-comfyui-api.hf.space{image_path}")
+
+    except Exception as e:
+        raise ValueError(f"Failed to save image: {e}")
+
+def make_request(url, data=None, headers=None):
+    req = urllib.request.Request(url, data=data, headers=headers)
+    try:
+        with urllib.request.urlopen(req) as response:
+            response_body = response.read().decode()  # Decode the response
+            # print(response_body)
+            return json.loads(response_body)  # Convert to JSON if valid
+    except urllib.error.HTTPError as e:
+        print(f"HTTPError: {e.code}, {e.reason}")
+        print(e.read().decode())  # Print detailed error response
+    except urllib.error.URLError as e:
+        print(f"URLError: {e.reason}")
+
+# Helper: Queue the prompt
+def queue_prompt(workflow, token):
+    payload = {"prompt": workflow, "client_id": client_id}
+    headers = {
+        'Authorization': f'Bearer {token}',
+        'Content-Type': 'application/json'
+    }
+    response = make_request(f"{server_address}/prompt", data=json.dumps(payload).encode('utf-8'), headers=headers)
+    if not response or 'prompt_id' not in response:
+        raise ValueError("Failed to queue the prompt. Check the request or API response.")
+    return response['prompt_id']
+
+def get_history(prompt_id, token):
+    headers = {
+        'Authorization': f'Bearer {token}',
+        'Content-Type': 'application/json'
+    }
+    return make_request(f"{server_address}/history/{prompt_id}", headers=headers)
+
+def get_video_data(filename, subfolder, token):
+    url_values = {'filename': filename, 'subfolder': subfolder}
+    url = f"{server_address}/view?{urllib.parse.urlencode(url_values)}"
+    req = urllib.request.Request(url)
+    req.add_header("Authorization", f"Bearer {token}")
+
+    try:
+        return urllib.request.urlopen(req).read()  # Return video content
+    except urllib.error.HTTPError as e:
+        print(f"HTTP Error: {e.code} - {e.reason}")
+        print(e.read())
+        raise
+
+# Helper: Upload Image to Hugging Face
+def upload_image_to_hf(image_path, token):
+    with open(image_path, 'rb') as img:
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "image/jpeg"  # Change as per your image type
+        }
+        response = requests.post(
+            "https://gosign-de-image-to-video.hf.space",
+            headers=headers,
+            files={'file': img}
+        )
+
+    if response.status_code != 200:
+        raise ValueError(f"Failed to upload image: {response.text}")
+
+    return response.json().get('url')
+
+# Route: Image to Video
+@app.route('/image_to_video', methods=['POST'])
+def image_to_video():
     data = request.json
 
-    # Extract token from headers
+    # Extract and validate token
     token = request.headers.get('Authorization')
-    if token is None:
-        return jsonify({'error': 'No token provided'}), 400
-    if token.startswith("Bearer "):
-        token = token.split(" ")[1]
-
-    # token = base64.b64decode(token).decode("utf-8")
+    if not token or not token.startswith("Bearer "):
+        return jsonify({'error': 'Valid Bearer token required'}), 400
+    token = token.split(" ")[1]
 
-    # Extract text prompt
-    text_prompt = data['text_prompt']
+    # Validate text prompt
+    text_prompt = data.get('text_prompt')
     if not text_prompt:
         return jsonify({'error': 'Text prompt is required'}), 400
 
-    # Handle uploaded image or base64-encoded image
+    # Handle uploaded image or base64 image
     image_file = request.files.get('image')
-    base64_image = data['base64_image']
+    base64_image = data.get('base64_image')
 
     if image_file:
-        # Validate and save uploaded image
+        # Validate and save the uploaded image
         if not allowed_file(image_file.filename):
             return jsonify({'error': 'Unsupported image format'}), 400
         filename = secure_filename(image_file.filename)
         image_path = f"/tmp/{uuid.uuid4()}_{filename}"
-        image_file.save(image_path)
 
+
+        image_file.save(image_path)
     elif base64_image:
-        # Save base64-encoded image
+        # Save base64 image
         try:
             image_path = save_base64_image(base64_image)
+            # return jsonify({'image_path': image_path})
+            # image_path = (upload_image_to_hf(image_path, token))
+
+            print(f"Image saved at HF: {image_path}")
+
         except Exception as e:
             return jsonify({'error': f'Invalid base64 image data: {str(e)}'}), 400
     else:
-        return jsonify({'error': 'Image is required (either file or base64)'}), 400
+        return jsonify({'error': 'Image is required (file or base64)'}), 400
 
-    # Get the path to the workflow configuration file
+    # Load workflow configuration
     current_dir = os.path.dirname(os.path.abspath(__file__))
-    file_path = os.path.join(current_dir, 'workflows/cogvideox_image_to_video_workflow_api.json')
-
-    print(f"Modified workflow: {file_path}", flush=True)
+    workflow_path = os.path.join(current_dir, 'workflows/cogvideox_image_to_video_workflow_api.json')
+    with open(workflow_path, 'r', encoding='utf-8') as f:
+        workflow = json.load(f)
 
+    # Modify workflow with inputs
+    workflow["30"]["inputs"]["prompt"] = text_prompt
+    workflow["36"]["inputs"]["image"] = image_path
+    workflow["31"]["inputs"]["prompt"] = "Low quality, watermark, strange motion"
+    workflow["57"]["inputs"]["seed"] = random.randint(100000000000000, 999999999999999)
 
-    # Load and modify workflow
-    with open(file_path, 'r', encoding='utf-8') as file:
-        workflow = json.load(file)
-    workflow["30"]["inputs"]["prompt"] = text_prompt  # Text prompt
-    workflow["36"]["inputs"]["upload"] = image_path   # Image path
-    workflow["31"]["inputs"]["prompt"] = "Low quality, watermark, strange motion"  # Negative prompt
-
-    seed = random.randint(1e14, 9e14)
-    workflow["57"]["inputs"]["seed"] = seed  # Set reproducibility seed
-
-    # WebSocket connection to trigger workflow
+    # WebSocket connection to queue and monitor workflow
     ws = websocket.WebSocket()
-    ws.connect(f"{ws_address}?clientId={client_id}&token={token}", header=
-        {"Authorization": f"Bearer {token}"})
-    ws.send(json.dumps({"workflow": workflow}))  # Send the modified workflow
+    try:
+        ws.connect(f"{ws_address}?clientId={client_id}&token={token}", header={"Authorization": f"Bearer {token}"})
+    except websocket.WebSocketException as e:
+        return jsonify({'error': f'WebSocket connection failed: {str(e)}'}), 500
 
-    # Receive video processing result
+    try:
+        prompt_id = queue_prompt(workflow, token)
+    except ValueError as e:
+        return jsonify({'error': str(e)}), 400
+
+    # Wait for workflow execution to complete
     while True:
-        out = ws.recv()
-        message = json.loads(out)
-        if message.get('type') == 'completed':
-            video_data = message['data']
+        message = json.loads(ws.recv())
+        if message.get('type') == 'executing' and message['data']['node'] is None and message['data']['prompt_id'] == prompt_id:
             break
 
-    # Fetch and return the generated video
-    video_content = fetch_video(video_data, token)
+    # Fetch video details
+    history = get_history(prompt_id, token).get(prompt_id, {})
+    video_data = None
+    for node_id, node_output in history.get('outputs', {}).items():
+        if 'videos' in node_output:
+            video = node_output['videos'][0]
+            video_data = get_video_data(video['filename'], video['subfolder'], token)
+
+    if not video_data:
+        return jsonify({'error': 'Failed to generate video'}), 500
+
+    # Save and return the generated video
+    local_video_path = f"/tmp/generated_video_{uuid.uuid4()}.mp4"
+    with open(local_video_path, 'wb') as f:
+        f.write(video_data)
+
     return send_file(
-        io.BytesIO(video_content),
+        io.BytesIO(video_data),
         mimetype='video/mp4',
         as_attachment=True,
         download_name='generated_video.mp4'
     )
 
+
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=7860)  # Removed 'debug=True'
+    app.run(host='0.0.0.0', port=7860, debug=True)  # Removed 'debug=True'

requirements.txt

@@ -5,3 +5,4 @@ websocket-client
 Pillow
 uuid
 gunicorn
+requests

workflows/cogvideox_image_to_video_workflow_api.json

@@ -54,7 +54,7 @@
   },
   "36": {
     "inputs": {
-      "image": "Webimage-1-720x480.jpg",
+      "image": "https://huggingface.co/spaces/gosign-de/comfyui-api/raw/main/images/Webimage-1-720x480.jpg",
       "upload": "image"
     },
     "class_type": "LoadImage",
@@ -131,8 +131,8 @@
       "steps": 50,
       "cfg": 6,
       "seed": 65334758276105,
-      "scheduler": "DPM",
-      "denoise_strength": 16,
+      "scheduler": "DPM++",
+      "denoise_strength": 1.0,
       "pipeline": [
         "1",
         0