Update core/app.py

core/app.py

@@ -4,14 +4,38 @@
 Refactored modular architecture – orchestrates specialised components
 """
 from __future__ import annotations
+
 # ── Early env/threading hygiene (safe default to silence libgomp) ────────────
 import os
 os.environ["OMP_NUM_THREADS"] = "2"  # Force valid value early
+
+# --- Stable runtime env defaults and optional secret-name mapping ---
+def _map_env(src: str, dst: str):
+    v = os.getenv(src)
+    if v and not os.getenv(dst):
+        os.environ[dst] = v
+
+# If HF blocks underscores in secret names, you can add simple names and map them:
+# Add secrets later like: OMPNUMTHREADS=1, TOKENIZERSPARALLELISM=false,
+# PYTORCHCUDAALLOCCONF=max_split_size_mb:128, LOGLEVEL=info, APPENV=production
+_map_env("OMPNUMTHREADS", "OMP_NUM_THREADS")
+_map_env("TOKENIZERSPARALLELISM", "TOKENIZERS_PARALLELISM")
+_map_env("PYTORCHCUDAALLOCCONF", "PYTORCH_CUDA_ALLOC_CONF")
+_map_env("LOGLEVEL", "LOG_LEVEL")
+_map_env("APPENV", "APP_ENV")
+
+# Critical defaults (safe even if already set elsewhere)
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+os.environ.setdefault("LOG_LEVEL", "info")
+os.environ.setdefault("APP_ENV", "production")
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "max_split_size_mb:128")
+
 # If you use early_env in your project, keep this import (harmless if absent)
 try:
-    import early_env # sets OMP/MKL/OPENBLAS + torch threads safely
+    import early_env  # sets OMP/MKL/OPENBLAS + torch threads safely
 except Exception:
     pass
+
 import logging
 import threading
 import traceback
@@ -19,24 +43,29 @@
 import time
 from pathlib import Path
 from typing import Optional, Tuple, Dict, Any, Callable
+
 # Mitigate CUDA fragmentation (must be set before importing torch)
 if "PYTORCH_CUDA_ALLOC_CONF" not in os.environ:
     os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True,max_split_size_mb:128"
+
 # ── Logging ──────────────────────────────────────────────────────────────────
 logging.basicConfig(
     level=logging.INFO,
     format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
 )
 logger = logging.getLogger("core.app")
+
 # ── Ensure project root importable ───────────────────────────────────────────
 PROJECT_FILE = Path(__file__).resolve()
 CORE_DIR = PROJECT_FILE.parent
 ROOT = CORE_DIR.parent
 if str(ROOT) not in sys.path:
     sys.path.insert(0, str(ROOT))
+
 # Create loader directories if they don't exist
 loaders_dir = ROOT / "models" / "loaders"
 loaders_dir.mkdir(parents=True, exist_ok=True)
+
 # ── Gradio schema patch (HF quirk) ───────────────────────────────────────────
 try:
     import gradio_client.utils as gc_utils
@@ -52,6 +81,7 @@ def _patched_get_type(schema):
     logger.info("Gradio schema patch applied")
 except Exception as e:
     logger.warning(f"Gradio patch failed: {e}")
+
 # ── Core config + components ─────────────────────────────────────────────────
 try:
     from config.app_config import get_config
@@ -61,8 +91,10 @@ class DummyConfig:
         def to_dict(self):
             return {}
     get_config = lambda: DummyConfig()
+
 from utils.hardware.device_manager import DeviceManager
 from utils.system.memory_manager import MemoryManager
+
 # Try to import the new split loaders first, fall back to old if needed
 try:
     from models.loaders.model_loader import ModelLoader
@@ -70,20 +102,23 @@ def to_dict(self):
 except ImportError:
     logger.warning("Split loaders not found, using legacy loader")
     # Fall back to old loader if split architecture isn't available yet
-    from models.model_loader import ModelLoader # type: ignore
+    from models.model_loader import ModelLoader  # type: ignore
+
 from processing.video.video_processor import CoreVideoProcessor
 from processing.audio.audio_processor import AudioProcessor
 from utils.monitoring.progress_tracker import ProgressTracker
 from utils.cv_processing import validate_video_file
+
 # ── Optional Two-Stage import ────────────────────────────────────────────────
 TWO_STAGE_AVAILABLE = False
 TWO_STAGE_IMPORT_ORIGIN = ""
 TWO_STAGE_IMPORT_ERROR = ""
 CHROMA_PRESETS: Dict[str, Dict[str, Any]] = {"standard": {}}
-TwoStageProcessor = None # type: ignore
+TwoStageProcessor = None  # type: ignore
+
 # Try multiple import paths for two-stage processor
 two_stage_paths = [
-    "processors.two_stage", # Your fixed version
+    "processors.two_stage",  # Your fixed version
     "processing.two_stage.two_stage_processor",
     "processing.two_stage",
 ]
@@ -97,19 +132,23 @@ def to_dict(self):
     except Exception as e:
         TWO_STAGE_IMPORT_ERROR = str(e)
         continue
+
 if not TWO_STAGE_AVAILABLE:
     logger.warning(f"Two-stage import FAILED from all paths: {TWO_STAGE_IMPORT_ERROR}")
+
 # ── Quiet startup self-check (async by default) ──────────────────────────────
 # Place the helper in tools/startup_selfcheck.py (with tools/__init__.py present)
 try:
     from tools.startup_selfcheck import schedule_startup_selfcheck
 except Exception:
-    schedule_startup_selfcheck = None # graceful if the helper isn't shipped
+    schedule_startup_selfcheck = None  # graceful if the helper isn't shipped
+
 # Dummy exceptions if core.exceptions not available
 class ModelLoadingError(Exception):
     pass
 class VideoProcessingError(Exception):
     pass
+
 # ╔══════════════════════════════════════════════════════════════════════════╗
 # ║ VideoProcessor class ║
 # ╚══════════════════════════════════════════════════════════════════════════╝
@@ -119,7 +158,7 @@ class VideoProcessor:
     """
     def __init__(self):
         self.config = get_config()
-        self._patch_config_defaults(self.config) # avoid AttributeError on older configs
+        self._patch_config_defaults(self.config)  # avoid AttributeError on older configs
         self.device_manager = DeviceManager()
         self.memory_manager = MemoryManager(self.device_manager.get_optimal_device())
         self.model_loader = ModelLoader(self.device_manager, self.memory_manager)
@@ -131,6 +170,7 @@ def __init__(self):
         self.cancel_event = threading.Event()
         self.progress_tracker: Optional[ProgressTracker] = None
         logger.info(f"VideoProcessor on device: {self.device_manager.get_optimal_device()}")
+
     # ── Config hardening: add missing fields safely ───────────────────────────
     @staticmethod
     def _patch_config_defaults(cfg: Any) -> None:
@@ -154,6 +194,7 @@ def _patch_config_defaults(cfg: Any) -> None:
             if not hasattr(cfg, k):
                 setattr(cfg, k, v)
         Path(cfg.output_dir).mkdir(parents=True, exist_ok=True)
+
     # ── Progress helper ───────────────────────────────────────────────────────
     def _init_progress(self, video_path: str, cb: Optional[Callable] = None):
         try:
@@ -167,6 +208,7 @@ def _init_progress(self, video_path: str, cb: Optional[Callable] = None):
         except Exception as e:
             logger.warning(f"Progress init failed: {e}")
             self.progress_tracker = ProgressTracker(100, cb)
+
     # ── Model loading ─────────────────────────────────────────────────────────
     def load_models(self, progress_callback: Optional[Callable] = None) -> str:
         with self.loading_lock:
@@ -181,11 +223,14 @@ def load_models(self, progress_callback: Optional[Callable] = None) -> str:
                 )
                 if self.cancel_event.is_set():
                     return "Model loading cancelled"
+
                 # Get the actual models
                 sam2_predictor = sam2_loaded.model if sam2_loaded else None
-                mat_model = mat_loaded.model if mat_loaded else None # NOTE: in our MatAnyone loader this is a stateful adapter (callable)
+                mat_model = mat_loaded.model if mat_loaded else None  # NOTE: stateful callable adapter
+
                 # Initialize core processor
                 self.core_processor = CoreVideoProcessor(config=self.config, models=self.model_loader)
+
                 # Initialize two-stage processor if available
                 self.two_stage_processor = None
                 if TWO_STAGE_AVAILABLE and TwoStageProcessor and (sam2_predictor or mat_model):
@@ -197,20 +242,21 @@ def load_models(self, progress_callback: Optional[Callable] = None) -> str:
                     except Exception as e:
                         logger.warning(f"Two-stage init failed: {e}")
                         self.two_stage_processor = None
+
                 self.models_loaded = True
                 msg = self.model_loader.get_load_summary()
-               
+
                 # Add status about processors
                 if self.two_stage_processor:
                     msg += "\n✅ Two-stage processor ready"
                 else:
                     msg += "\n⚠️ Two-stage processor not available"
-                   
+
                 if mat_model:
                     msg += "\n✅ MatAnyone refinement active"
                 else:
                     msg += "\n⚠️ MatAnyone not loaded (edges may be rough)"
-                   
+
                 logger.info(msg)
                 return msg
             except (AttributeError, ModelLoadingError) as e:
@@ -223,6 +269,7 @@ def load_models(self, progress_callback: Optional[Callable] = None) -> str:
                 err = f"Unexpected error during model loading: {e}"
                 logger.error(f"{err}\n{traceback.format_exc()}")
                 return err
+
     # ── Public entry – process video ─────────────────────────────────────────
     def process_video(
         self,
@@ -236,20 +283,19 @@ def process_video(
         preview_mask: bool = False,
         preview_greenscreen: bool = False,
     ) -> Tuple[Optional[str], Optional[str], str]:
-        
         # ===== BACKGROUND PATH DEBUG & FIX =====
         logger.info("=" * 60)
         logger.info("BACKGROUND PATH DEBUGGING")
         logger.info(f"background_choice: {background_choice}")
         logger.info(f"custom_background_path type: {type(custom_background_path)}")
         logger.info(f"custom_background_path value: {custom_background_path}")
-        
+
         # Fix 1: Handle if Gradio sends a dict
         if isinstance(custom_background_path, dict):
             original = custom_background_path
             custom_background_path = custom_background_path.get('name') or custom_background_path.get('path')
             logger.info(f"Extracted path from dict: {original} -> {custom_background_path}")
-        
+
         # Fix 2: Handle PIL Image objects
         try:
             from PIL import Image
@@ -261,7 +307,7 @@ def process_video(
                     logger.info(f"Saved PIL Image to: {custom_background_path}")
         except ImportError:
             pass
-        
+
         # Fix 3: Verify file exists when using custom background
         if background_choice == "custom" or custom_background_path:
             if custom_background_path:
@@ -273,7 +319,7 @@ def process_video(
                     import glob
                     patterns = [
                         "/tmp/gradio*/**/*.jpg",
-                        "/tmp/gradio*/**/*.jpeg", 
+                        "/tmp/gradio*/**/*.jpeg",
                         "/tmp/gradio*/**/*.png",
                         "/tmp/**/*.jpg",
                         "/tmp/**/*.jpeg",
@@ -292,25 +338,29 @@ def process_video(
                                 break
             else:
                 logger.error("❌ Custom background mode but path is None!")
-        
+
         logger.info(f"Final custom_background_path: {custom_background_path}")
         logger.info("=" * 60)
-        
+
         if not self.models_loaded or not self.core_processor:
             return None, None, "Models not loaded. Please click 'Load Models' first."
         if self.cancel_event.is_set():
             return None, None, "Processing cancelled"
+
         self._init_progress(video_path, progress_callback)
         ok, why = validate_video_file(video_path)
         if not ok:
             return None, None, f"Invalid video: {why}"
+
         try:
             # Log which mode we're using
             mode = "two-stage" if use_two_stage else "single-stage"
             matanyone_status = "enabled" if self.model_loader.get_matanyone() else "disabled"
             logger.info(f"Processing video in {mode} mode, MatAnyone: {matanyone_status}")
+
             # IMPORTANT: start each video with a clean MatAnyone memory
             self._reset_matanyone_session()
+
             if use_two_stage:
                 if not TWO_STAGE_AVAILABLE or self.two_stage_processor is None:
                     return None, None, "Two-stage processing not available"
@@ -339,6 +389,7 @@ def process_video(
         except Exception as e:
             logger.error(f"Unexpected processing error: {e}\n{traceback.format_exc()}")
             return None, None, f"Unexpected error: {e}"
+
     # ── Private – per-video MatAnyone reset ──────────────────────────────────
     def _reset_matanyone_session(self):
         """
@@ -355,6 +406,7 @@ def _reset_matanyone_session(self):
                 logger.info("MatAnyone session reset for new video")
             except Exception as e:
                 logger.warning(f"MatAnyone session reset failed (continuing): {e}")
+
     # ── Private – single-stage ───────────────────────────────────────────────
     def _process_single_stage(
         self,
@@ -365,15 +417,15 @@ def _process_single_stage(
         preview_mask: bool,
         preview_greenscreen: bool,
     ) -> Tuple[Optional[str], Optional[str], str]:
-        
         # Additional debug logging for single-stage
         logger.info(f"[Single-stage] background_choice: {background_choice}")
         logger.info(f"[Single-stage] custom_background_path: {custom_background_path}")
-        
+
         ts = int(time.time())
         out_dir = Path(self.config.output_dir) / "single_stage"
         out_dir.mkdir(parents=True, exist_ok=True)
         out_path = str(out_dir / f"processed_{ts}.mp4")
+
         # Process video via your CoreVideoProcessor
         result = self.core_processor.process_video(
             input_path=video_path,
@@ -384,9 +436,10 @@ def _process_single_stage(
             },
             progress_callback=progress_callback,
         )
-       
+
         if not result:
             return None, None, "Video processing failed"
+
         # Mux audio unless preview-only
         if not (preview_mask or preview_greenscreen):
             try:
@@ -398,6 +451,7 @@ def _process_single_stage(
                 final_path = out_path
         else:
             final_path = out_path
+
         # Build status message
         try:
             mat_loaded = bool(self.model_loader.get_matanyone())
@@ -413,6 +467,7 @@ def _process_single_stage(
             f"Device: {self.device_manager.get_optimal_device()}"
         )
         return final_path, None, msg  # No green in single-stage
+
     # ── Private – two-stage ─────────────────────────────────────────────────
     def _process_two_stage(
         self,
@@ -425,11 +480,11 @@ def _process_two_stage(
     ) -> Tuple[Optional[str], Optional[str], str]:
         if self.two_stage_processor is None:
             return None, None, "Two-stage processor not available"
-        
+
         # Additional debug logging for two-stage
         logger.info(f"[Two-stage] background_choice: {background_choice}")
         logger.info(f"[Two-stage] custom_background_path: {custom_background_path}")
-        
+
         import cv2
         cap = cv2.VideoCapture(video_path)
         if not cap.isOpened():
@@ -437,6 +492,7 @@ def _process_two_stage(
         w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) or 1280
         h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) or 720
         cap.release()
+
         # Prepare background
         try:
             background = self.core_processor.prepare_background(
@@ -447,12 +503,15 @@ def _process_two_stage(
             return None, None, f"Failed to prepare background: {e}"
         if background is None:
             return None, None, "Failed to prepare background"
+
         ts = int(time.time())
         out_dir = Path(self.config.output_dir) / "two_stage"
         out_dir.mkdir(parents=True, exist_ok=True)
         final_out = str(out_dir / f"final_{ts}.mp4")
+
         chroma_cfg = CHROMA_PRESETS.get(chroma_preset, CHROMA_PRESETS.get("standard", {}))
         logger.info(f"Two-stage with preset: {chroma_preset} | key_color: {key_color_mode}")
+
         # (Per-video reset already called in process_video)
         final_path, green_path, stage2_msg = self.two_stage_processor.process_full_pipeline(
             video_path,
@@ -464,6 +523,7 @@ def _process_two_stage(
         )
         if final_path is None:
             return None, None, stage2_msg
+
         # Mux audio
         try:
             final_with_audio = self.audio_processor.add_audio_to_video(
@@ -472,6 +532,7 @@ def _process_two_stage(
         except Exception as e:
             logger.warning(f"Audio mux failed: {e}")
             final_with_audio = final_path
+
         try:
             mat_loaded = bool(self.model_loader.get_matanyone())
         except Exception:
@@ -485,6 +546,7 @@ def _process_two_stage(
             f"Device: {self.device_manager.get_optimal_device()}"
         )
         return final_with_audio, green_path, msg
+
     # ── Status helpers ───────────────────────────────────────────────────────
     def get_status(self) -> Dict[str, Any]:
         status = {
@@ -496,7 +558,7 @@ def get_status(self) -> Dict[str, Any]:
             "config": self._safe_config_dict(),
             "memory_usage": self._safe_memory_usage(),
         }
-       
+
         try:
             status["sam2_loaded"] = self.model_loader.get_sam2() is not None
             status["matanyone_loaded"] = self.model_loader.get_matanyone() is not None
@@ -504,10 +566,12 @@ def get_status(self) -> Dict[str, Any]:
         except Exception:
             status["sam2_loaded"] = False
             status["matanyone_loaded"] = False
+
         if self.progress_tracker:
             status["progress"] = self.progress_tracker.get_all_progress()
-           
+
         return status
+
     def _safe_config_dict(self) -> Dict[str, Any]:
         try:
             return self.config.to_dict()
@@ -516,14 +580,17 @@ def _safe_config_dict(self) -> Dict[str, Any]:
                     "ffmpeg_path", "max_model_size", "max_model_size_bytes",
                     "output_dir", "matanyone_enabled"]
             return {k: getattr(self.config, k, None) for k in keys}
+
     def _safe_memory_usage(self) -> Dict[str, Any]:
         try:
             return self.memory_manager.get_memory_usage()
         except Exception:
             return {}
+
     def cancel_processing(self):
         self.cancel_event.set()
         logger.info("Cancellation requested")
+
     def cleanup_resources(self):
         try:
             self.memory_manager.cleanup_aggressive()
@@ -534,10 +601,13 @@ def cleanup_resources(self):
         except Exception:
             pass
         logger.info("Resources cleaned up")
+
 # ── Singleton + thin wrappers (used by UI callbacks) ────────────────────────
 processor = VideoProcessor()
+
 def load_models_with_validation(progress_callback: Optional[Callable] = None) -> str:
     return processor.load_models(progress_callback)
+
 def process_video_fixed(
     video_path: str,
     background_choice: str,
@@ -560,38 +630,42 @@ def process_video_fixed(
         preview_mask,
         preview_greenscreen,
     )
+
 def get_model_status() -> Dict[str, Any]:
     return processor.get_status()
+
 def get_cache_status() -> Dict[str, Any]:
     return processor.get_status()
+
 PROCESS_CANCELLED = processor.cancel_event
+
 # ── CLI entrypoint (must exist; app.py imports main) ─────────────────────────
 def main():
     try:
         logger.info("Starting BackgroundFX Pro")
         logger.info(f"Device: {processor.device_manager.get_optimal_device()}")
         logger.info(f"Two-stage available: {TWO_STAGE_AVAILABLE}")
-        
+
         # 🔹 Quiet model self-check (defaults to async; set SELF_CHECK_MODE=sync to block)
         if schedule_startup_selfcheck is not None:
             try:
                 schedule_startup_selfcheck(mode=os.getenv("SELF_CHECK_MODE", "async"))
             except Exception as e:
                 logger.error(f"Startup self-check skipped: {e}", exc_info=True)
-        
+
         # Log model loader type
         try:
             from models.loaders.model_loader import ModelLoader
             logger.info("Using split loader architecture")
         except Exception:
             logger.info("Using legacy loader")
-        
+
         # FIXED: Move UI import inside main() to avoid circular dependency
         # and add better error handling
         try:
             # Import here to break circular dependency
             from ui import ui_components
-            
+
             # Now get the create_interface function
             if hasattr(ui_components, 'create_interface'):
                 create_interface = ui_components.create_interface
@@ -599,12 +673,12 @@ def main():
                 logger.error("create_interface not found in ui_components")
                 logger.error(f"Available attributes: {dir(ui_components)}")
                 raise ImportError("create_interface function not found")
-                
+
         except ImportError as e:
             logger.error(f"Failed to import UI components: {e}")
             import traceback
             traceback.print_exc()
-            
+
             # Try alternate import method
             try:
                 logger.info("Trying alternate import method...")
@@ -616,7 +690,7 @@ def main():
                 logger.error(f"Alternate import also failed: {e2}")
                 logger.info("System initialized but UI unavailable. Exiting.")
                 return
-        
+
         # Create and launch the interface
         try:
             demo = create_interface()
@@ -630,7 +704,7 @@ def main():
             logger.error(f"Failed to launch Gradio interface: {e}")
             import traceback
             traceback.print_exc()
-            
+
     except Exception as e:
         logger.error(f"Fatal error in main: {e}")
         import traceback