from flask import Flask, request, jsonify from flask_cors import CORS import os import base64 from PIL import Image import io import torch from transformers import CLIPProcessor, CLIPModel from mlc_llm import ChatModule import threading import numpy as np import hashlib import time app = Flask(__name__) CORS(app) # Enable CORS for all routes # --- Configuration for MLC LLM --- # IMPORTANT: Before running, ensure you have downloaded and compiled an MLC LLM model. # Example: # 1. Install MLC LLM: pip install mlc-llm-nightly -f https://mlc.ai/wheels # 2. Download a model: mlc_llm chat Llama-2-7b-chat-hf-q4f16_1 --model-path ./model_artifacts # (This will create a `dist` folder inside `model_artifacts` with the model.) # 3. Update MLC_MODEL_PATH and MLC_MODEL_NAME below to match your downloaded model. # E.g., if you run `mlc_llm chat Llama-2-7b-chat-hf-q4f16_1`, it will create a folder # like `Llama-2-7b-chat-hf-q4f16_1` within your specified --model-path. MLC_MODEL_ARTIFACTS_DIR = os.getenv("MLC_MODEL_ARTIFACTS_DIR", "./backend/model_artifacts") MLC_MODEL_NAME = os.getenv("MLC_MODEL_NAME", "Llama-2-7b-chat-hf-q4f16_1") MLC_MODEL_PATH = os.path.join(MLC_MODEL_ARTIFACTS_DIR, MLC_MODEL_NAME) # --- Configuration for HuggingFace CLIP --- CLIP_MODEL_NAME = "openai/clip-vit-base-patch32" # Global instances for models clip_processor = None clip_model = None mlc_chat_module = None mlc_lock = threading.Lock() # To ensure thread-safe access to the LLM def load_mlc_llm_model(): global mlc_chat_module if mlc_chat_module is None: print(f"Attempting to load LLM model: {MLC_MODEL_NAME} from {MLC_MODEL_PATH}...") try: if not os.path.exists(MLC_MODEL_PATH): print(f"Error: MLC LLM model path not found: {MLC_MODEL_PATH}") print("Please ensure the MLC LLM model is downloaded and compiled in the specified path.") print("Refer to installation instructions for mlc-llm and model download commands.") return None mlc_chat_module = ChatModule(model=MLC_MODEL_NAME, model_path=MLC_MODEL_PATH) print("MLC LLM model loaded successfully.") except Exception as e: print(f"Error loading MLC LLM model: {e}") mlc_chat_module = None return mlc_chat_module def load_clip_model(): global clip_processor, clip_model if clip_model is None: print(f"Attempting to load CLIP model: {CLIP_MODEL_NAME}...") try: clip_processor = CLIPProcessor.from_pretrained(CLIP_MODEL_NAME) clip_model = CLIPModel.from_pretrained(CLIP_MODEL_NAME) # Move model to GPU if available for faster inference if torch.cuda.is_available(): clip_model.to("cuda") print("CLIP model moved to CUDA.") print("CLIP model loaded successfully.") except Exception as e: print(f"Error loading CLIP model: {e}") clip_processor, clip_model = None, None return clip_processor, clip_model # Load models on app startup with app.app_context(): if mlc_chat_module is None: load_mlc_llm_model() if clip_model is None: load_clip_model() @app.route('/') def health_check(): llm_status = "loaded" if mlc_chat_module else "not loaded (check logs)" clip_status = "loaded" if clip_model else "not loaded (check logs)" return jsonify({ "status": "Quantum-Enhanced WAN 2.1 Backend is running!", "llm_status": llm_status, "clip_status": clip_status }) @app.route('/embed_image', methods=['POST']) def embed_image(): if clip_processor is None or clip_model is None: return jsonify({"error": "CLIP model not loaded. Check server logs for details."}), 500 data = request.get_json() image_data_url = data.get('image') if not image_data_url: return jsonify({"error": "No image data provided"}), 400 try: header, encoded = image_data_url.split(",", 1) image_bytes = base64.b64decode(encoded) image = Image.open(io.BytesIO(image_bytes)).convert("RGB") inputs = clip_processor(images=image, return_tensors="pt") if torch.cuda.is_available(): inputs = {k: v.to("cuda") for k, v in inputs.items()} with torch.no_grad(): image_features = clip_model.get_image_features(**inputs) # Normalize embeddings and convert to list for JSON serialization image_embeddings = image_features / image_features.norm(p=2, dim=-1, keepdim=True) return jsonify({"embeddings": image_embeddings.squeeze().tolist()}) except Exception as e: print(f"Error embedding image: {e}") return jsonify({"error": f"Failed to embed image: {str(e)}"}), 500 @app.route('/chat/completions', methods=['POST']) def chat_completions_endpoint(): if mlc_chat_module is None: return jsonify({"error": "LLM model not loaded. Check server logs for details."}), 500 data = request.get_json() prompt = data.get("prompt") system_message = data.get("system_message", "You are a creative AI assistant for video generation.") if not prompt: return jsonify({"error": "Prompt is required"}), 400 try: full_prompt = f"{system_message}\nUser: {prompt}" with mlc_lock: mlc_chat_module.reset_chat() response = mlc_chat_module.generate(full_prompt) return jsonify({"completion": response}) except Exception as e: print(f"Error getting chat completion: {e}") return jsonify({"error": f"Failed to get chat completion: {str(e)}"}), 500 @app.route('/generate_frame_guidance', methods=['POST']) def generate_frame_guidance(): # This endpoint provides LLM guidance for the frontend's quantum diffusion. # It does NOT generate the image itself. if mlc_chat_module is None or clip_processor is None or clip_model is None: return jsonify({"error": "One or more AI models not loaded. Check server logs for details."}), 500 data = request.get_json() image_data_url = data.get('image') # The current frame from the frontend prompt = data.get('prompt', 'Quantum interpolation') influence = data.get('influence', 5) # 0-100 entanglement_depth = data.get('depth', 16) # For LLM to consider frame_number = data.get('frame_number', 0) if not image_data_url: return jsonify({"error": "No image data provided"}), 400 try: # 1. Get CLIP embeddings for the current frame header, encoded = image_data_url.split(",", 1) image_bytes = base64.b64decode(encoded) input_image = Image.open(io.BytesIO(image_bytes)).convert("RGB") clip_inputs = clip_processor(images=input_image, return_tensors="pt") if torch.cuda.is_available(): clip_inputs = {k: v.to("cuda") for k, v in clip_inputs.items()} with torch.no_grad(): image_features = clip_model.get_image_features(**clip_inputs) image_embeddings_np = image_features.squeeze().cpu().numpy() embedding_snippet = ", ".join([f"{x:.4f}" for x in image_embeddings_np[:10]]) # 2. Use LLM to generate guidance for the next quantum diffusion step llm_prompt = ( f"You are an AI video director for a quantum diffusion system. Your task is to guide the transformation " f"of a video frame based on quantum principles and user input. " f"Given the current frame's visual context (CLIP features: [{embedding_snippet}...]), " f"the user's creative prompt: '{prompt}', " f"and the quantum settings (Quantum Influence: {influence}%, Entanglement Depth: {entanglement_depth} layers), " f"describe *precisely* how the quantum diffusion effect should transform the current frame into frame {frame_number + 1}. " f"Think of these transformations as manipulating a quantum state that manifests visually. " f"Higher influence and depth should lead to more pronounced, chaotic, or surreal quantum effects. " f"Focus on quantifiable visual parameters, including: " f"color shifts (e.g., 'shift red by +{Math.round(influence/5)}', 'hue rotate {Math.round(influence*1.5)}deg'), " f"blur (e.g., 'apply gaussian blur radius {Math.max(1, Math.round(influence/10))}'), " f"glitch/distortion (e.g., 'pixel displacement x-axis random {Math.max(5, Math.round(influence/5))}px', 'chromatic aberration offset {Math.max(1, Math.round(influence/20))}'), " f"zoom/pan (e.g., 'zoom in {1.00 + influence/2000}x, pan right {Math.round(influence/10)}px'), " f"pattern overlay (e.g., 'overlay subtle static pattern opacity {influence/200}'), " f"motion blur (e.g., 'apply motion blur strength {Math.round(entanglement_depth/2)}'), " f"bloom (e.g., 'add bloom strength {influence/100}'), " f"noise (e.g., 'add noise amount {influence/50}'), " f"vignette (e.g., 'add vignette strength {influence/200}'), " f"or specific quantum-themed visual cues (e.g., 'ripple effect', 'add subtle scanlines opacity {influence/200}', 'invert colors'). " f"Combine these to create a dynamic, quantum-like visual evolution. Ensure the intensity of effects scales with Influence and Depth. " f"Be concise and output only the transformation instructions. " f"Example: 'shift blue by +{Math.round(influence/5)}, apply motion blur strength {Math.round(entanglement_depth/2)}, zoom {1.00 + influence/2000}x, add subtle scanlines opacity {influence/200}'.\n" f"Transformation Instructions for frame {frame_number + 1}:" ) llm_guidance = "" try: with mlc_lock: mlc_chat_module.reset_chat() llm_guidance = mlc_chat_module.generate(llm_prompt) except Exception as llm_e: print(f"LLM guidance generation failed: {llm_e}. Using fallback guidance.") llm_guidance = f"apply subtle glitch effect, shift colors slightly based on quantum influence {influence}%." print(f"LLM Guidance: {llm_guidance}") return jsonify({ "guidance": llm_guidance, "log": (f"Backend provided guidance for frame {frame_number + 1} based on prompt: '{prompt[:50]}...', " f"influence: {influence}, depth: {entanglement_depth}. LLM guidance: '{llm_guidance[:50]}...'.") }) except Exception as e: print(f"Error generating frame guidance: {e}") return jsonify({"error": f"Failed to generate frame guidance: {str(e)}"}), 500 if __name__ == '__main__': if not os.path.exists(MLC_MODEL_ARTIFACTS_DIR): os.makedirs(MLC_MODEL_ARTIFACTS_DIR) print(f"Created model artifacts directory: {MLC_MODEL_ARTIFACTS_DIR}") app.run(debug=True, host='0.0.0.0', port=5000)