Add optional HF Hub cloud storage for videos with auto folders per module

.env.example

@@ -59,4 +59,13 @@ GEMINI_API_KEY=your_gemini_key_here
 # Groq API Key (Primary for AI script generation)
 # Get from: https://console.groq.com/keys
 GROQ_API=gsk_your_groq_key_here
+
+# ===================
+# HF Cloud Storage (Optional - Recommended)
+# ===================
+# If set, videos will upload to HF Hub Dataset repo
+# Format: username/repo-name (e.g., robiul487/ncakit-videos)
+# Uses HF_TOKEN for authentication (already set above)
+# Folders auto-created: short_video/, story_reels/
+HF_REPO=your-username/ncakit-videos
 ```

app.py

@@ -65,6 +65,17 @@ async def startup_event():
     # Ensure directories exist
     config.ensure_directories()
     
+    # Initialize HF Hub storage (optional)
+    if config.hf_repo:
+        from modules.shared.services.hf_storage import init_hf_storage
+        import os
+        hf_token = os.getenv("HF_TOKEN")
+        hf_storage = init_hf_storage(repo_id=config.hf_repo, token=hf_token)
+        if hf_storage.enabled:
+            logger.info(f"HF Cloud storage enabled: {config.hf_repo}")
+        else:
+            logger.warning("HF credentials set but storage failed to initialize")
+    
     # Register all modules
     num_modules = registry.register_all(app, config)
     logger.info(f"Loaded {num_modules} module(s)")

config.py

@@ -78,6 +78,11 @@ class NCAkitConfig(BaseConfig):
     gemini_api_key: Optional[str] = None  # For AI script generation (fallback)
     groq_api: Optional[str] = None  # Groq API key (primary for script generation)
     
+    # ===================
+    # HF Cloud Storage (Optional)
+    # ===================
+    hf_repo: Optional[str] = None  # HF repo for videos (e.g., "username/ncakit-videos")
+    # Note: Uses HF_TOKEN from environment for auth
     @property
     def videos_dir_path(self) -> Path:
         """Directory for storing generated videos"""

modules/shared/__init__.py

@@ -0,0 +1,3 @@
+"""
+Shared utilities and services for NCAkit modules
+"""

modules/shared/services/__init__.py

@@ -0,0 +1,6 @@
+"""
+Shared services for NCAkit modules
+"""
+from .hf_storage import HFStorageClient, get_hf_storage, init_hf_storage
+
+__all__ = ['HFStorageClient', 'get_hf_storage', 'init_hf_storage']

modules/shared/services/hf_storage.py

@@ -0,0 +1,169 @@
+"""
+HF Hub Storage Client - Optional cloud storage for videos
+Uploads videos to Hugging Face Hub Dataset repository
+"""
+import logging
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# Will be imported if available
+try:
+    from huggingface_hub import HfApi, create_repo
+    HF_HUB_AVAILABLE = True
+except ImportError:
+    HF_HUB_AVAILABLE = False
+    logger.warning("huggingface_hub not installed. Cloud storage disabled.")
+
+
+class HFStorageClient:
+    """
+    Optional HF Hub storage for videos.
+    
+    If HF_REPO and HF_TOKEN are set, uploads videos to HF Dataset repo.
+    Otherwise, does nothing (graceful degradation).
+    
+    Structure:
+    - your-repo/
+      - short_video/
+        - video_id.mp4
+      - story_reels/
+        - video_id.mp4
+      - [future_module]/
+        - video_id.mp4
+    """
+    
+    def __init__(self, repo_id: str = None, token: str = None):
+        """
+        Initialize HF storage client.
+        
+        Args:
+            repo_id: HF repo ID (e.g., "username/ncakit-videos")
+            token: HF token with write access
+        """
+        self.enabled = bool(repo_id and token and HF_HUB_AVAILABLE)
+        self.repo_id = repo_id
+        self.token = token
+        self.api = None
+        
+        if self.enabled:
+            self._initialize()
+        else:
+            if repo_id and token and not HF_HUB_AVAILABLE:
+                logger.warning("HF credentials provided but huggingface_hub not installed")
+            elif not repo_id or not token:
+                logger.info("HF cloud storage disabled (HF_REPO or HF_TOKEN not set)")
+    
+    def _initialize(self):
+        """Initialize HF API and create repo if needed"""
+        try:
+            self.api = HfApi(token=self.token)
+            
+            # Create repo if not exists
+            try:
+                create_repo(
+                    repo_id=self.repo_id,
+                    repo_type="dataset",
+                    token=self.token,
+                    exist_ok=True,
+                    private=False  # Public for easy access
+                )
+                logger.info(f"HF storage enabled: {self.repo_id}")
+            except Exception as e:
+                logger.warning(f"Could not create/verify repo: {e}")
+                
+        except Exception as e:
+            logger.error(f"Failed to initialize HF storage: {e}")
+            self.enabled = False
+    
+    def upload_video(
+        self, 
+        local_path: Path, 
+        video_id: str, 
+        folder: str = "videos"
+    ) -> Optional[str]:
+        """
+        Upload video to HF Hub.
+        
+        Args:
+            local_path: Path to local video file
+            video_id: Unique video ID
+            folder: Folder name (e.g., "short_video", "story_reels")
+            
+        Returns:
+            Public URL if successful, None otherwise
+        """
+        if not self.enabled:
+            return None
+        
+        if not local_path.exists():
+            logger.error(f"Video file not found: {local_path}")
+            return None
+        
+        try:
+            # Upload file
+            path_in_repo = f"{folder}/{video_id}.mp4"
+            
+            self.api.upload_file(
+                path_or_fileobj=str(local_path),
+                path_in_repo=path_in_repo,
+                repo_id=self.repo_id,
+                repo_type="dataset"
+            )
+            
+            # Generate public URL
+            public_url = f"https://huggingface.co/datasets/{self.repo_id}/resolve/main/{path_in_repo}"
+            
+            logger.info(f"Uploaded to HF: {public_url}")
+            return public_url
+            
+        except Exception as e:
+            logger.error(f"Failed to upload to HF Hub: {e}")
+            return None
+    
+    def delete_video(self, video_id: str, folder: str = "videos") -> bool:
+        """
+        Delete video from HF Hub.
+        
+        Args:
+            video_id: Video ID to delete
+            folder: Folder name
+            
+        Returns:
+            True if successful
+        """
+        if not self.enabled:
+            return False
+        
+        try:
+            path_in_repo = f"{folder}/{video_id}.mp4"
+            
+            self.api.delete_file(
+                path_in_repo=path_in_repo,
+                repo_id=self.repo_id,
+                repo_type="dataset"
+            )
+            
+            logger.info(f"Deleted from HF: {path_in_repo}")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to delete from HF Hub: {e}")
+            return False
+
+
+# Singleton instance (initialized in modules/__init__.py or app startup)
+_hf_storage: Optional[HFStorageClient] = None
+
+
+def get_hf_storage() -> Optional[HFStorageClient]:
+    """Get the global HF storage client"""
+    return _hf_storage
+
+
+def init_hf_storage(repo_id: str = None, token: str = None) -> HFStorageClient:
+    """Initialize the global HF storage client"""
+    global _hf_storage
+    _hf_storage = HFStorageClient(repo_id=repo_id, token=token)
+    return _hf_storage

modules/story_reels/services/story_creator.py

@@ -473,11 +473,31 @@ class StoryCreator:
             )
             
             job["video_url"] = f"/videos/{job_id}.mp4"
+            job["storage"] = "local"
             job["duration"] = audio_duration
             job["progress"] = 100
             
             logger.info(f"[{job_id}] Video saved to {output_path}")
             
+            # Optional: Upload to HF Hub (if configured)
+            try:
+                from modules.shared.services.hf_storage import get_hf_storage
+                hf_storage = get_hf_storage()
+                if hf_storage and hf_storage.enabled:
+                    cloud_url = hf_storage.upload_video(
+                        local_path=output_path,
+                        video_id=job_id,
+                        folder="story_reels"  # Module-specific folder
+                    )
+                    if cloud_url:
+                        job["video_url"] = cloud_url
+                        job["storage"] = "cloud"
+                        # Delete local file to save Docker space
+                        output_path.unlink()
+                        logger.info(f"[{job_id}] Uploaded to HF Hub, local file deleted")
+            except Exception as e:
+                logger.warning(f"[{job_id}] HF upload failed, keeping local: {e}")
+            
         finally:
             # Cleanup temp files
             for temp_file in temp_files:

modules/video_creator/services/short_creator.py

@@ -327,6 +327,24 @@ class ShortCreator:
             if temp_output_path.exists():
                 temp_output_path.rename(output_path)
                 logger.info(f"Video {video_id} created successfully at {output_path}")
+                
+                # Optional: Upload to HF Hub (if configured)
+                try:
+                    from modules.shared.services.hf_storage import get_hf_storage
+                    hf_storage = get_hf_storage()
+                    if hf_storage and hf_storage.enabled:
+                        cloud_url = hf_storage.upload_video(
+                            local_path=output_path,
+                            video_id=video_id,
+                            folder="short_video"  # Module-specific folder
+                        )
+                        if cloud_url:
+                            # Store cloud URL for status response
+                            # Note: video_creator uses file-based status, so we just log
+                            output_path.unlink()
+                            logger.info(f"Video {video_id} uploaded to HF Hub, local file deleted")
+                except Exception as e:
+                    logger.warning(f"HF upload failed for {video_id}, keeping local: {e}")
             else:
                 raise Exception("Rendered file not found at temp path")
                 

requirements.txt

@@ -23,3 +23,4 @@ groq
 
 # Utilities
 python-multipart
+huggingface_hub