Update app.py

app.py

@@ -11,8 +11,9 @@ import zipfile
 import uvicorn
 from typing import Dict, List, Set, Optional
 from fastapi import FastAPI, BackgroundTasks, HTTPException, Form
-from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.responses import HTMLResponse, JSONResponse, FileResponse
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
 from huggingface_hub import HfApi, list_repo_files
 from huggingface_hub.utils import HfHubHTTPError
 
@@ -20,17 +21,17 @@ from huggingface_hub.utils import HfHubHTTPError
 HF_TOKEN = os.getenv("HF_TOKEN", "")
 SOURCE_REPO_ID = os.getenv("SOURCE_REPO", "Fred808/BG1") # Source for RARs
 DEST_REPO_ID_RAR = os.getenv("DEST_REPO_RAR", "") # Destination for extracted RAR contents (set to empty string if not needed)
-DEST_REPO_ID_VIDEO = os.getenv("DEST_REPO_VIDEO", "Fred808/BG3") # Destination for zipped video frames
+# DEST_REPO_ID_VIDEO = os.getenv("DEST_REPO_VIDEO", "Fred808/BG3") # Destination for zipped video frames - DISABLED FOR DOWNLOAD MODE
 
 DOWNLOAD_FOLDER = "downloads"
 EXTRACT_FOLDER = "extracted_tmp"
-VIDEO_FRAMES_EXTRACT_FOLDER = "video_frames_tmp"
-ZIPPED_FRAMES_FOLDER = "zipped_frames"
+VIDEO_FRAMES_EXTRACT_FOLDER = "video_frames"  # Changed to keep frames permanently
+# ZIPPED_FRAMES_FOLDER = "zipped_frames"  # No longer needed since we're not zipping
 
 DOWNLOAD_STATE_FILE = "download_progress.json"
 PROCESS_STATE_FILE = "process_progress.json"
 UPLOADED_FOLDERS_FILE = "uploaded_folders.json"  # Track uploaded folder hashes for BG2
-PROCESSED_VIDEO_COURSES_FILE = "processed_video_courses.json" # Track processed video course folders for BG3
+PROCESSED_VIDEO_COURSES_FILE = "processed_video_courses.json" # Track processed video course folders
 FAILED_FILES_LOG = "failed_files.txt"
 
 CHUNK_SIZE = 3  # Smaller chunks for Space environment
@@ -40,7 +41,6 @@ VIDEO_FRAME_FPS = 3 # Frames per second to extract from videos
 os.makedirs(DOWNLOAD_FOLDER, exist_ok=True)
 os.makedirs(EXTRACT_FOLDER, exist_ok=True)
 os.makedirs(VIDEO_FRAMES_EXTRACT_FOLDER, exist_ok=True)
-os.makedirs(ZIPPED_FRAMES_FOLDER, exist_ok=True)
 
 api = HfApi(token=HF_TOKEN)
 
@@ -52,12 +52,12 @@ processing_status = {
     "processed_files": 0,
     "failed_files": 0,
     "uploaded_rar_folders": 0,
-    "uploaded_video_courses": 0,
+    "extracted_video_courses": 0,  # Changed from uploaded_video_courses to extracted_video_courses
     "last_update": None,
     "logs": []
 }
 
-app = FastAPI(title="RAR & Video Processing Service", description="Automated RAR extraction and video frame upload service")
+app = FastAPI(title="RAR & Video Processing Service", description="Automated RAR extraction and video frame extraction service with download capability")
 
 # Add CORS middleware
 app.add_middleware(
@@ -107,7 +107,7 @@ def save_uploaded_folders(uploaded_set: Set[str]):
         json.dump({"uploaded_folder_hashes": list(uploaded_set)}, f)
 
 def load_processed_video_courses() -> Set[str]:
-    """Loads the set of processed video course folder names for BG3."""
+    """Loads the set of processed video course folder names."""
     if os.path.exists(PROCESSED_VIDEO_COURSES_FILE):
         try:
             with open(PROCESSED_VIDEO_COURSES_FILE, "r") as f:
@@ -118,7 +118,7 @@ def load_processed_video_courses() -> Set[str]:
     return set()
 
 def save_processed_video_courses(processed_set: set):
-    """Saves the set of processed video course folder names for BG3 to a file."""
+    """Saves the set of processed video course folder names to a file."""
     with open(PROCESSED_VIDEO_COURSES_FILE, "w") as f:
         json.dump(list(processed_set), f)
 
@@ -227,22 +227,6 @@ def extract_frames(video_path: str, output_folder: str, fps: int) -> bool:
         log_message(f"❌ Error extracting frames from {os.path.basename(video_path)}: {e.stderr}")
         return False
 
-def zip_folder(folder_path: str, output_zip_path: str) -> bool:
-    """Zips the contents of a folder."""
-    log_message(f" compressing {folder_path} to {output_zip_path}...")
-    try:
-        with zipfile.ZipFile(output_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
-            for root, _, files in os.walk(folder_path):
-                for file in files:
-                    file_path = os.path.join(root, file)
-                    arcname = os.path.relpath(file_path, folder_path)
-                    zipf.write(file_path, arcname)
-        log_message(f"✅ Successfully zipped {folder_path}")
-        return True
-    except Exception as e:
-        log_message(f"❌ Error zipping {folder_path}: {e}")
-        return False
-
 def upload_file_to_hf(local_path: str, path_in_repo: str, repo_id: str, max_retries: int = 5, initial_delay: int = 5) -> bool:
     """Uploads a single file to Hugging Face Hub with retry logic and exponential backoff."""
     log_message(f"⬆️ Uploading {os.path.basename(local_path)} to {repo_id}/{path_in_repo}")
@@ -271,15 +255,15 @@ def upload_file_to_hf(local_path: str, path_in_repo: str, repo_id: str, max_retr
     log_message(f"❌ Failed to upload {os.path.basename(local_path)} after {max_retries} attempts.")
     return False
 
-def process_video_frames_and_upload(extracted_rar_folder: str, processed_video_courses_set: Set[str]) -> bool:
-    """Scans an extracted RAR folder for MP4s, extracts frames, zips, and uploads to BG3."""
+def process_video_frames_for_download(extracted_rar_folder: str, processed_video_courses_set: Set[str]) -> bool:
+    """Scans an extracted RAR folder for MP4s, extracts frames, and saves them for download (no zipping)."""
     video_processed_successfully = False
     
     # Use the top-level folder name of the extracted RAR as the course folder name
     course_folder_name = os.path.basename(extracted_rar_folder)
 
     if course_folder_name in processed_video_courses_set:
-        log_message(f"⏩ Video frames for course \'{course_folder_name}\' already processed. Skipping.")
+        log_message(f"⏩ Video frames for course '{course_folder_name}' already processed. Skipping.")
         return True
 
     log_message(f"🎬 Processing videos in extracted RAR folder: {course_folder_name}")
@@ -314,45 +298,32 @@ def process_video_frames_and_upload(extracted_rar_folder: str, processed_video_c
 
     # Check if any frames were extracted for the entire course folder
     if frames_extracted_count == 0:
-        log_message(f"⚠️ No frames extracted for any video in {course_folder_name}. Skipping zipping and upload to BG3.")
+        log_message(f"⚠️ No frames extracted for any video in {course_folder_name}.")
         if os.path.exists(course_video_extract_dir):
             shutil.rmtree(course_video_extract_dir)
         return False
 
-    course_zip_path = os.path.join(ZIPPED_FRAMES_FOLDER, f"{course_folder_name}_frames.zip")
-    if zip_folder(course_video_extract_dir, course_zip_path):
-        path_in_repo = f"{course_folder_name}_frames.zip"
-        if upload_file_to_hf(course_zip_path, path_in_repo, DEST_REPO_ID_VIDEO):
-            log_message(f"✅ Successfully processed video frames and uploaded {course_folder_name}_frames.zip to BG3")
-            processed_video_courses_set.add(course_folder_name) # Mark as processed
-            save_processed_video_courses(processed_video_courses_set) # Save state
-            video_processed_successfully = True
-        else:
-            log_message(f"❌ Failed to upload zipped video frames for {course_folder_name} to BG3")
-    else:
-        log_message(f"❌ Failed to zip video frames for {course_folder_name}")
-
-    # Cleanup local video frame files
-    log_message(f"🧹 Cleaning up local video frame files for {course_folder_name}")
-    if os.path.exists(course_video_extract_dir):
-        shutil.rmtree(course_video_extract_dir)
-    if os.path.exists(course_zip_path):
-        os.remove(course_zip_path)
+    # No zipping - just keep the frames in the folder for direct access
+    log_message(f"✅ Successfully processed video frames for {course_folder_name} - frames available for download")
+    processed_video_courses_set.add(course_folder_name) # Mark as processed
+    save_processed_video_courses(processed_video_courses_set) # Save state
+    video_processed_successfully = True
 
     return video_processed_successfully
 
 def extract_and_upload_rar(rar_path: str, processed_rars_set: set, uploaded_folders_set: Set[str], processed_video_courses_set: Set[str]) -> bool:
-    """Extracts a single RAR file, uploads its contents to BG2 (if DEST_REPO_ID_RAR is set), and then processes videos for BG3"""
+    """Extracts a single RAR file, uploads its contents to BG2 (if DEST_REPO_ID_RAR is set), and then processes videos for download"""
     filename = os.path.basename(rar_path)
     processing_status["current_file"] = filename
 
     folder_name = filename.replace(".rar", "")
+    folder_hash = get_folder_hash(folder_name)
     current_extract_folder = os.path.join(EXTRACT_FOLDER, f"{folder_name}_extracted")
 
     # Check if RAR is already processed (uploaded to BG2 or video frames processed)
     # This logic needs to be careful. If BG2 is not set, we only care about video processing.
     # If video processing is not needed, we only care about BG2 upload.
-    is_bg2_processed = (not DEST_REPO_ID_RAR) or (get_folder_hash(folder_name) in uploaded_folders_set)
+    is_bg2_processed = (not DEST_REPO_ID_RAR) or (folder_hash in uploaded_folders_set)
     is_bg3_processed = (folder_name in processed_video_courses_set)
 
     if filename in processed_rars_set and is_bg2_processed and is_bg3_processed:
@@ -361,8 +332,8 @@ def extract_and_upload_rar(rar_path: str, processed_rars_set: set, uploaded_fold
 
     # If BG2 upload is enabled and folder already uploaded to BG2, skip RAR extraction/upload to BG2
     # but still proceed to video processing if not already done.
-    if DEST_REPO_ID_RAR and get_folder_hash(folder_name) in uploaded_folders_set and not is_bg3_processed:
-        log_message(f"🔒 Folder \'{folder_name}\' already uploaded to BG2 (hash: {get_folder_hash(folder_name)[:8]}...), skipping RAR upload.")
+    if DEST_REPO_ID_RAR and folder_hash in uploaded_folders_set and not is_bg3_processed:
+        log_message(f"🔒 Folder '{folder_name}' already uploaded to BG2 (hash: {folder_hash[:8]}...), skipping RAR upload.")
         # If the extracted folder doesn't exist, we can't process videos from it.
         # This scenario might happen if the previous run was interrupted after BG2 upload but before video processing cleanup.
         if not os.path.exists(current_extract_folder): 
@@ -371,7 +342,7 @@ def extract_and_upload_rar(rar_path: str, processed_rars_set: set, uploaded_fold
         else:
             # Proceed to video processing if not already done
             log_message(f"Continuing with video processing for {filename}.")
-            video_processed = process_video_frames_and_upload(current_extract_folder, processed_video_courses_set)
+            video_processed = process_video_frames_for_download(current_extract_folder, processed_video_courses_set)
             if video_processed:
                 processed_rars_set.add(filename)
                 save_processed_files_state(processed_rars_set)
@@ -433,18 +404,22 @@ def extract_and_upload_rar(rar_path: str, processed_rars_set: set, uploaded_fold
                 uploaded_folders_set.add(folder_hash)
                 save_uploaded_folders(uploaded_folders_set)
                 processing_status["uploaded_rar_folders"] = len(uploaded_folders_set)
-                log_message(f"🔒 Folder \'{folder_name}\' locked in BG2 repo (hash: {folder_hash[:8]}...)")
+                log_message(f"🔒 Folder '{folder_name}' locked in BG2 repo (hash: {folder_hash[:8]}...)")
             else:
                 log_message(f"⚠️ No files were successfully uploaded from {filename} to BG2.")
         else:
             log_message("Skipping upload to BG2 as DEST_REPO_ID_RAR is not set.")
         
-        # Now process video frames from the extracted content and upload to BG3
-        video_processed = process_video_frames_and_upload(current_extract_folder, processed_video_courses_set)
+        # Now process video frames from the extracted content for download
+        video_processed = process_video_frames_for_download(current_extract_folder, processed_video_courses_set)
 
-        # Mark RAR as processed only if both BG2 (if enabled) and BG3 processing are successful
-        # Or if BG2 is not enabled, only BG3 processing needs to be successful
+        # Mark RAR as processed only if both BG2 (if enabled) and video processing are successful
+        # Or if BG2 is not enabled, only video processing needs to be successful
         if (not DEST_REPO_ID_RAR or (folder_hash in uploaded_folders_set)) and video_processed:
+            processed_rars_set.add(filename)
+            save_processed_files_state(processed_rars_set)
+            processing_status["processed_files"] = len(processed_rars_set)
+            processing_status["extracted_video_courses"] = len(processed_video_courses_set)
             return True
         elif DEST_REPO_ID_RAR and not (folder_hash in uploaded_folders_set):
             log_message(f"❌ RAR processing failed for {filename}: BG2 upload was not successful.")
@@ -487,9 +462,9 @@ def continuous_processing(start_download_index: Optional[int] = None):
         uploaded_folders = load_uploaded_folders()
         processing_status["uploaded_rar_folders"] = len(uploaded_folders)
 
-        # Load processed video courses tracking for BG3
+        # Load processed video courses tracking
         processed_video_courses = load_processed_video_courses()
-        processing_status["uploaded_video_courses"] = len(processed_video_courses)
+        processing_status["extracted_video_courses"] = len(processed_video_courses)
         
         if start_download_index is not None:
             log_message(f"Starting download from index: {start_download_index}")
@@ -532,9 +507,6 @@ def continuous_processing(start_download_index: Optional[int] = None):
                 filename = os.path.basename(rar_file_path)
                 success = extract_and_upload_rar(rar_file_path, processed_rars, uploaded_folders, processed_video_courses)
                 if success:
-                    # processed_rars.add(filename) is handled inside extract_and_upload_rar now for better atomicity
-                    # processing_status["processed_files"] += 1 is also handled inside
-                    
                     # Delete the RAR file after successful processing
                     log_message(f"🗑️ Deleting processed RAR: {filename}")
                     try:
@@ -557,6 +529,9 @@ def continuous_processing(start_download_index: Optional[int] = None):
         processing_status["current_file"] = None
         log_message("🏁 Processing stopped")
 
+# Mount the video frames directory as static files for direct access
+app.mount("/frames", StaticFiles(directory=VIDEO_FRAMES_EXTRACT_FOLDER), name="frames")
+
 @app.get("/", response_class=HTMLResponse)
 async def root():
     """Serve the main HTML interface"""
@@ -577,18 +552,26 @@ async def root():
             .button:disabled { background: #ccc; cursor: not-allowed; }
             .stop-button { background: #f44336; }
             .stop-button:hover { background: #d32f2f; }
+            .download-button { background: #4CAF50; }
+            .download-button:hover { background: #45a049; }
             .stats { display: flex; gap: 20px; margin: 20px 0; }
             .stat-item { background: #f0f0f0; padding: 10px; border-radius: 5px; text-align: center; flex: 1; }
             .start-form { margin-top: 20px; padding: 15px; border: 1px solid #ddd; border-radius: 5px; background: #f9f9f9; }
             .start-form input[type=\"number\"] { width: calc(100% - 120px); padding: 8px; margin-right: 10px; border: 1px solid #ccc; border-radius: 4px; }
             .start-form button { padding: 8px 15px; background: #4CAF50; color: white; border: none; border-radius: 4px; cursor: pointer; }
             .start-form button:hover { background: #45a049; }
+            .downloads-section { margin-top: 30px; padding: 20px; border: 1px solid #ddd; border-radius: 5px; background: #f9f9f9; }
+            .download-list { max-height: 300px; overflow-y: auto; }
+            .download-item { display: flex; justify-content: space-between; align-items: center; padding: 10px; border-bottom: 1px solid #eee; }
+            .download-item:last-child { border-bottom: none; }
+            .folder-link { color: #2196F3; text-decoration: none; }
+            .folder-link:hover { text-decoration: underline; }
         </style>
     </head>
     <body>
         <div class=\"container\">
             <h1>🔄 RAR & Video Processing Service</h1>
-            <p>Automated extraction and upload of RAR files from BG1 to BG2 dataset, and video frame extraction/upload to BG3 dataset</p>
+            <p>Automated extraction and upload of RAR files from BG1 to BG2 dataset, and video frame extraction for download</p>
             
             <div class=\"status-card\">
                 <h3>Status: <span id=\"status\">Stopped</span></h3>
@@ -610,8 +593,8 @@ async def root():
                     <span id=\"uploaded-rar-folders\">0</span>
                 </div>
                 <div class=\"stat-item\">
-                    <h4>Uploaded Video Courses (BG3)</h4>
-                    <span id=\"uploaded-video-courses\">0</span>
+                    <h4>Extracted Video Courses</h4>
+                    <span id=\"extracted-video-courses\">0</span>
                 </div>
                 <div class=\"stat-item\">
                     <h4>Failed</h4>
@@ -629,6 +612,15 @@ async def root():
                 <button class=\"button\" onclick=\"startProcessing()\" id=\"start-btn\">Start Processing (from last saved index)</button>
                 <button class=\"button stop-button\" onclick=\"stopProcessing()\" id=\"stop-btn\" disabled>Stop Processing</button>
                 <button class=\"button\" onclick=\"refreshStatus()\">Refresh Status</button>
+                <button class=\"button download-button\" onclick=\"refreshDownloads()\">Refresh Downloads</button>
+            </div>
+            
+            <div class=\"downloads-section\">
+                <h3>Available Frame Folders</h3>
+                <p>Click on folder names to browse extracted video frames directly</p>
+                <div class=\"download-list\" id=\"download-list\">
+                    <p>Loading...</p>
+                </div>
             </div>
             
             <h3>Logs</h3>
@@ -689,7 +681,7 @@ async def root():
                     document.getElementById(\"total-files\").textContent = status.total_files;
                     document.getElementById(\"processed-files\").textContent = status.processed_files;
                     document.getElementById(\"uploaded-rar-folders\").textContent = status.uploaded_rar_folders;
-                    document.getElementById(\"uploaded-video-courses\").textContent = status.uploaded_video_courses;
+                    document.getElementById(\"extracted-video-courses\").textContent = status.extracted_video_courses;
                     document.getElementById(\"failed-files\").textContent = status.failed_files;
                     
                     document.getElementById(\"start-btn\").disabled = status.is_running;
@@ -703,11 +695,35 @@ async def root():
                 }
             }
             
+            async function refreshDownloads() {
+                try {
+                    const response = await fetch(\"/frame-folders\");
+                    const folders = await response.json();
+                    
+                    const downloadList = document.getElementById(\"download-list\");
+                    if (folders.folders.length === 0) {
+                        downloadList.innerHTML = \"<p>No frame folders available yet.</p>\";
+                    } else {
+                        downloadList.innerHTML = folders.folders.map(folder => 
+                            `<div class=\"download-item\">
+                                <span>${folder.name} (${folder.video_count} videos, ${folder.frame_count} frames)</span>
+                                <a href=\"/frames/${folder.name}/\" class=\"folder-link\" target=\"_blank\">Browse Frames</a>
+                            </div>`
+                        ).join(\"\");
+                    }
+                } catch (error) {
+                    console.error(\"Error refreshing downloads:\", error);
+                    document.getElementById(\"download-list\").innerHTML = \"<p>Error loading frame folders.</p>\";
+                }
+            }
+            
             // Auto-refresh every 5 seconds
             setInterval(refreshStatus, 5000);
+            setInterval(refreshDownloads, 10000);
             
             // Initial load
             refreshStatus();
+            refreshDownloads();
         </script>
     </body>
     </html>
@@ -762,11 +778,47 @@ async def get_uploaded_folders():
 
 @app.get("/processed-video-courses")
 async def get_processed_video_courses():
-    """Get list of processed video course folder names for BG3"""
+    """Get list of processed video course folder names"""
     processed_video_courses = load_processed_video_courses()
     return {"processed_video_course_count": len(processed_video_courses), "course_names": list(processed_video_courses)}
 
+@app.get("/frame-folders")
+async def list_frame_folders():
+    """List available frame folders with statistics"""
+    try:
+        if not os.path.exists(VIDEO_FRAMES_EXTRACT_FOLDER):
+            return {"folders": []}
+        
+        folders = []
+        for folder_name in os.listdir(VIDEO_FRAMES_EXTRACT_FOLDER):
+            folder_path = os.path.join(VIDEO_FRAMES_EXTRACT_FOLDER, folder_name)
+            if os.path.isdir(folder_path):
+                # Count video folders and total frames
+                video_count = 0
+                frame_count = 0
+                
+                for video_folder in os.listdir(folder_path):
+                    video_folder_path = os.path.join(folder_path, video_folder)
+                    if os.path.isdir(video_folder_path):
+                        video_count += 1
+                        # Count frames in this video folder
+                        frame_files = [f for f in os.listdir(video_folder_path) if f.lower().endswith(('.png', '.jpg', '.jpeg'))]
+                        frame_count += len(frame_files)
+                
+                folders.append({
+                    "name": folder_name,
+                    "video_count": video_count,
+                    "frame_count": frame_count,
+                    "path": folder_path
+                })
+        
+        # Sort by folder name
+        folders.sort(key=lambda x: x["name"])
+        return {"folders": folders}
+    except Exception as e:
+        log_message(f"❌ Error listing frame folders: {e}")
+        return {"folders": [], "error": str(e)}
+
 if __name__ == "__main__":
     uvicorn.run(app, host="0.0.0.0", port=7860)
 
-