Update app.py

app.py

@@ -1,5 +1,5 @@
 # =========================
-# app.py  (fixed version)
+# app.py  (Enhanced Version with Lip-Sync Optimization)
 # =========================
 import os
 
@@ -66,7 +66,7 @@ with main_col1:
         help="Frames are resized before detection/swap. Lower = faster."
     )
 
-    # NEW: Face blending percentage
+    # Face blending percentage
     face_blend_percent = st.sidebar.slider(
         "Face Blending Percentage", 
         min_value=0, 
@@ -83,6 +83,32 @@ with main_col1:
         help="Method for selecting which face to use from the source image"
     )
 
+    # NEW: Lip-sync specific settings
+    st.sidebar.markdown("---")
+    st.sidebar.subheader("🎭 Lip-Sync Optimization")
+    
+    lip_sync_enabled = st.sidebar.checkbox(
+        "Enable Lip-Sync Mode",
+        value=True,
+        help="Reduces glitches and blurriness in mouth movements"
+    )
+    
+    mouth_mask_strength = st.sidebar.slider(
+        "Mouth Mask Strength",
+        min_value=0,
+        max_value=100,
+        value=80,
+        help="How strongly to protect the mouth region from artifacts"
+    )
+    
+    frame_consistency = st.sidebar.slider(
+        "Frame Consistency",
+        min_value=0,
+        max_value=100,
+        value=70,
+        help="Maintain consistency between frames for smoother video"
+    )
+
     # For video mode only
     if st.session_state.mode == "video":
         fps_cap = st.sidebar.selectbox(
@@ -202,9 +228,9 @@ def _cv2_to_pil(image):
 def _pil_to_cv2(image):
     return cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
 
-# FIXED: Face blending function
-def _blend_faces(original_face, swapped_face, blend_percent):
-    """Blend between original and swapped faces based on percentage"""
+# Enhanced face blending function with mouth protection
+def _blend_faces(original_face, swapped_face, blend_percent, mouth_mask=None):
+    """Blend between original and swapped faces with optional mouth protection"""
     if blend_percent == 100:
         return swapped_face
     
@@ -213,7 +239,71 @@ def _blend_faces(original_face, swapped_face, blend_percent):
         swapped_face = cv2.resize(swapped_face, (original_face.shape[1], original_face.shape[0]))
     
     alpha = blend_percent / 100.0
-    return cv2.addWeighted(swapped_face, alpha, original_face, 1 - alpha, 0)
+    
+    if mouth_mask is not None:
+        # Apply different blending for mouth vs non-mouth regions
+        if mouth_mask.shape[:2] != original_face.shape[:2]:
+            mouth_mask = cv2.resize(mouth_mask, (original_face.shape[1], original_face.shape[0]))
+        
+        # Stronger blending for non-mouth areas, preserve mouth more
+        mouth_alpha = min(alpha + 0.2, 1.0)  # Less blending in mouth area
+        blended = np.zeros_like(original_face)
+        
+        # Mouth area - more preservation of swapped face
+        blended[mouth_mask > 0] = cv2.addWeighted(
+            swapped_face[mouth_mask > 0], mouth_alpha,
+            original_face[mouth_mask > 0], 1 - mouth_alpha, 0
+        )
+        
+        # Non-mouth area - normal blending
+        blended[mouth_mask == 0] = cv2.addWeighted(
+            swapped_face[mouth_mask == 0], alpha,
+            original_face[mouth_mask == 0], 1 - alpha, 0
+        )
+        
+        return blended
+    else:
+        # Standard blending if no mouth mask
+        return cv2.addWeighted(swapped_face, alpha, original_face, 1 - alpha, 0)
+
+# Create mouth mask from facial landmarks
+def _create_mouth_mask(face_landmarks, image_shape, strength=80):
+    """Create a mask focusing on the mouth region"""
+    if not hasattr(face_landmarks, 'landmark_2d_106'):
+        return None
+    
+    landmarks = face_landmarks.landmark_2d_106
+    if landmarks is None or len(landmarks) < 106:
+        return None
+    
+    # Mouth landmark indices (approximate for 106-point model)
+    mouth_indices = list(range(48, 68))  # Lips outline
+    mouth_inner = list(range(60, 68))    # Inner mouth
+    
+    if len(landmarks) < 68:
+        return None
+    
+    mask = np.zeros(image_shape[:2], dtype=np.uint8)
+    
+    # Create convex hull around mouth
+    mouth_points = np.array([landmarks[i] for i in mouth_indices], dtype=np.int32)
+    if len(mouth_points) > 2:
+        hull = cv2.convexHull(mouth_points)
+        cv2.fillPoly(mask, [hull], 255)
+    
+    # Add inner mouth with lower strength
+    inner_points = np.array([landmarks[i] for i in mouth_inner], dtype=np.int32)
+    if len(inner_points) > 2:
+        inner_hull = cv2.convexHull(inner_points)
+        cv2.fillPoly(mask, [inner_hull], 200)  # Lower value for inner area
+    
+    # Apply Gaussian blur for smooth edges
+    mask = cv2.GaussianBlur(mask, (15, 15), 0)
+    
+    # Adjust based on strength parameter
+    mask = cv2.addWeighted(mask, strength/100.0, np.zeros_like(mask), 0, 0)
+    
+    return mask
 
 # Face selection methods
 def _select_face(faces, method, image_shape=None):
@@ -235,11 +325,12 @@ def _select_face(faces, method, image_shape=None):
         return faces[0]
 
 # -------------------------------------
-# Core: FIXED face swap functions
+# Core: Enhanced face swap functions with lip-sync optimization
 # -------------------------------------
 def swap_faces_in_image(
     source_image_bgr, target_image_bgr, proc_res, max_faces, 
-    blend_percent=100, face_selection="Largest"
+    blend_percent=100, face_selection="Largest", lip_sync_enabled=True,
+    mouth_mask_strength=80
 ):
     # Get source face
     try:
@@ -286,7 +377,7 @@ def swap_faces_in_image(
             reverse=True
         )[:max_faces]
 
-        # Swap faces - FIXED implementation
+        # Swap faces with lip-sync optimization
         result_image = target_image_proc.copy()
         for tface in target_faces:
             try:
@@ -302,18 +393,18 @@ def swap_faces_in_image(
                 # Extract the face region
                 face_region = result_image[y1:y2, x1:x2].copy()
                 
-                # Perform the swap using the original method (paste_back=True)
-                swapped_region = swapper.get(result_image, tface, source_face, paste_back=True)
+                # Create mouth mask if lip-sync is enabled
+                mouth_mask = None
+                if lip_sync_enabled and hasattr(tface, 'landmark_2d_106'):
+                    mouth_mask = _create_mouth_mask(tface, face_region.shape, mouth_mask_strength)
                 
-                # Extract just the swapped face region
+                # Perform the swap
+                swapped_region = swapper.get(result_image, tface, source_face, paste_back=True)
                 swapped_face = swapped_region[y1:y2, x1:x2]
                 
-                # Apply blending if needed
-                if blend_percent < 100:
-                    blended_face = _blend_faces(face_region, swapped_face, blend_percent)
-                    result_image[y1:y2, x1:x2] = blended_face
-                else:
-                    result_image[y1:y2, x1:x2] = swapped_face
+                # Apply blending with mouth protection
+                blended_face = _blend_faces(face_region, swapped_face, blend_percent, mouth_mask)
+                result_image[y1:y2, x1:x2] = blended_face
                     
             except Exception as swap_e:
                 st.error(f"Face swap error: {swap_e}")
@@ -332,7 +423,8 @@ def swap_faces_in_image(
 
 def swap_faces_in_video(
     image_bgr, video_path, proc_res, fps_cap, keep_original_res, 
-    max_faces, blend_percent, face_selection, output_quality, progress
+    max_faces, blend_percent, face_selection, output_quality, progress,
+    lip_sync_enabled=True, mouth_mask_strength=80, frame_consistency=70
 ):
     # Get source face
     try:
@@ -382,12 +474,13 @@ def swap_faces_in_video(
 
     st.info(
         f"Processing: {proc_w}×{proc_h} | Output: {out_w}×{out_h} @ {write_fps:.2f} fps | "
-        f"Frame step: {frame_step} | Blend: {blend_percent}%"
+        f"Frame step: {frame_step} | Blend: {blend_percent}% | Lip-sync: {'ON' if lip_sync_enabled else 'OFF'}"
     )
 
     # Process loop
     read_idx = 0
     processed_frames = 0
+    previous_faces = {}  # For frame consistency
 
     try:
         while True:
@@ -415,6 +508,10 @@ def swap_faces_in_video(
                 except Exception as det_e:
                     target_faces = []
 
+                # Apply frame consistency
+                if frame_consistency > 0 and previous_faces and target_faces:
+                    target_faces = _apply_frame_consistency(target_faces, previous_faces, frame_consistency/100.0)
+
                 # Limit faces
                 if target_faces:
                     target_faces = sorted(
@@ -422,8 +519,9 @@ def swap_faces_in_video(
                         key=lambda f: (f.bbox[2]-f.bbox[0])*(f.bbox[3]-f.bbox[1]),
                         reverse=True
                     )[:max_faces]
+                    previous_faces = {i: face for i, face in enumerate(target_faces)}
 
-                # Swap faces - FIXED implementation
+                # Swap faces with lip-sync optimization
                 result_frame = proc_frame.copy()
                 for tface in target_faces:
                     try:
@@ -439,18 +537,18 @@ def swap_faces_in_video(
                         # Extract the face region
                         face_region = result_frame[y1:y2, x1:x2].copy()
                         
+                        # Create mouth mask if lip-sync is enabled
+                        mouth_mask = None
+                        if lip_sync_enabled and hasattr(tface, 'landmark_2d_106'):
+                            mouth_mask = _create_mouth_mask(tface, face_region.shape, mouth_mask_strength)
+                        
                         # Perform the swap
                         swapped_region = swapper.get(result_frame, tface, source_face, paste_back=True)
-                        
-                        # Extract just the swapped face region
                         swapped_face = swapped_region[y1:y2, x1:x2]
                         
-                        # Apply blending if needed
-                        if blend_percent < 100:
-                            blended_face = _blend_faces(face_region, swapped_face, blend_percent)
-                            result_frame[y1:y2, x1:x2] = blended_face
-                        else:
-                            result_frame[y1:y2, x1:x2] = swapped_face
+                        # Apply blending with mouth protection
+                        blended_face = _blend_faces(face_region, swapped_face, blend_percent, mouth_mask)
+                        result_frame[y1:y2, x1:x2] = blended_face
                             
                     except Exception as swap_e:
                         continue
@@ -484,6 +582,25 @@ def swap_faces_in_video(
 
     return output_path
 
+def _apply_frame_consistency(current_faces, previous_faces, consistency_strength):
+    """Maintain consistency between frames for smoother video"""
+    if not current_faces or not previous_faces:
+        return current_faces
+    
+    consistent_faces = []
+    for i, current_face in enumerate(current_faces):
+        if i in previous_faces:
+            # Blend current face with previous face for consistency
+            prev_face = previous_faces[i]
+            # Simple position smoothing (you can add more sophisticated blending)
+            current_face.bbox = [
+                prev_face.bbox[j] * consistency_strength + current_face.bbox[j] * (1 - consistency_strength)
+                for j in range(4)
+            ]
+        consistent_faces.append(current_face)
+    
+    return consistent_faces
+
 # -------------------------
 # UI: Improved layout
 # -------------------------
@@ -554,7 +671,8 @@ with main_col2:
                     output_path = swap_faces_in_video(
                         source_image, tmp_video_path, proc_res, fps_cap, 
                         keep_original_res, max_faces, face_blend_percent, 
-                        face_selection_method, output_quality, progress_bar
+                        face_selection_method, output_quality, progress_bar,
+                        lip_sync_enabled, mouth_mask_strength, frame_consistency
                     )
 
                 if output_path:
@@ -601,7 +719,8 @@ with main_col2:
                 with st.spinner("Processing image…"):
                     result_image = swap_faces_in_image(
                         source_image, target_image, proc_res, max_faces,
-                        face_blend_percent, face_selection_method
+                        face_blend_percent, face_selection_method,
+                        lip_sync_enabled, mouth_mask_strength
                     )
 
                 if result_image:
@@ -667,5 +786,6 @@ with st.expander("🩺 Diagnostics"):
         "- If you see errors: try different source/target images with clear faces\n"
         "- For better results: use high-quality images with front-facing faces\n"
         "- If processing is slow: reduce resolution or target FPS\n"
-        "- For videos: use MP4 format with H.264 encoding"
+        "- For videos: use MP4 format with H.264 encoding\n"
+        "- For best lip-sync: enable lip-sync mode and adjust mouth mask strength"
     )