Deploy DeepFake Detector API - 2026-04-20 01:46:18

app/services/model_registry.py

@@ -152,17 +152,23 @@ class ModelRegistry:
                     details={"repo_id": fusion_repo_id}
                 )
             
-            # Load submodels concurrently with a small bound to avoid
-            # overwhelming the container while still reducing cold-start wall time.
+            # Prepare submodels sequentially to avoid concurrent Hugging Face
+            # download contention, then load the already-downloaded artifacts in parallel.
+            prepared_submodels = []
+            for submodel_repo_id in submodel_repos:
+                prepared_submodels.append(
+                    await self._prepare_submodel(submodel_repo_id)
+                )
+
             max_concurrent_loads = 2
             semaphore = asyncio.Semaphore(max_concurrent_loads)
 
-            async def load_with_limit(repo_id: str):
+            async def load_with_limit(prepared_submodel):
                 async with semaphore:
-                    return await self._load_submodel(repo_id)
+                    return await self._load_prepared_submodel(prepared_submodel)
 
             load_results = await asyncio.gather(
-                *(load_with_limit(submodel_repo_id) for submodel_repo_id in submodel_repos),
+                *(load_with_limit(prepared_submodel) for prepared_submodel in prepared_submodels),
                 return_exceptions=True
             )
 
@@ -194,29 +200,39 @@ class ModelRegistry:
             self._is_loaded = True
             logger.info(f"Successfully loaded {len(self._submodels)} submodels and fusion model")
     
-    async def _load_submodel(self, repo_id: str) -> BaseSubmodelWrapper:
+    async def _prepare_submodel(self, repo_id: str) -> Dict[str, Any]:
         """
-        Download and load a single submodel.
-        
-        Uses the config to determine the correct wrapper class.
-        
-        Args:
-            repo_id: Hugging Face repository ID for the submodel
+        Download a submodel repository and prepare metadata for loading.
+
+        This stays sequential to avoid concurrent Hugging Face download issues.
         """
-        logger.info(f"Loading submodel: {repo_id}")
-        
-        # Download the repo
+        logger.info(f"Preparing submodel: {repo_id}")
+
         local_path = await asyncio.to_thread(
             self._hf_service.download_repo, repo_id
         )
-        
-        # Read config
         config = self._read_config(local_path)
-        
-        # Select appropriate wrapper class based on config
         wrapper_class = get_wrapper_class(config)
+
+        return {
+            "repo_id": repo_id,
+            "local_path": local_path,
+            "config": config,
+            "wrapper_class": wrapper_class,
+        }
+
+    async def _load_prepared_submodel(self, prepared_submodel: Dict[str, Any]) -> BaseSubmodelWrapper:
+        """
+        Load a submodel that has already been downloaded and prepared.
+        """
+        repo_id = prepared_submodel["repo_id"]
+        local_path = prepared_submodel["local_path"]
+        config = prepared_submodel["config"]
+        wrapper_class = prepared_submodel["wrapper_class"]
+
+        logger.info(f"Loading submodel: {repo_id}")
         logger.info(f"Using wrapper class {wrapper_class.__name__} for {repo_id}")
-        
+
         # Create and load wrapper
         wrapper = wrapper_class(
             repo_id=repo_id,