Update app.py

app.py

@@ -284,34 +284,39 @@ def process_video_frames_and_upload(extracted_rar_folder: str, processed_video_c
 
     log_message(f"🎬 Processing videos in extracted RAR folder: {course_folder_name}")
     
-    mp4_files_found = []
+    video_files_found = []
     for root, _, files in os.walk(extracted_rar_folder):
         for file in files:
-            if file.lower().endswith(".mp4"):
-                mp4_files_found.append(os.path.join(root, file))
+            # Check for common video file extensions
+            if file.lower().endswith(('.mp4', '.avi', '.mov', '.mkv', '.flv', '.wmv')):
+                video_files_found.append(os.path.join(root, file))
 
-    if not mp4_files_found:
-        log_message(f"⚠️ No MP4 files found in {course_folder_name}. Skipping video frame extraction.")
+    if not video_files_found:
+        log_message(f"⚠️ No video files found in {course_folder_name}. Skipping video frame extraction.")
         return False # Indicate no video processing was done
 
     course_video_extract_dir = os.path.join(VIDEO_FRAMES_EXTRACT_FOLDER, course_folder_name)
     os.makedirs(course_video_extract_dir, exist_ok=True)
 
-    for mp4_path in mp4_files_found:
-        video_basename = os.path.splitext(os.path.basename(mp4_path))[0]
+    frames_extracted_count = 0
+    for video_path in video_files_found:
+        video_basename = os.path.splitext(os.path.basename(video_path))[0]
         # Create a unique output folder for frames from this video within the course's frame directory
         video_output_folder = os.path.join(course_video_extract_dir, video_basename)
         
-        if not extract_frames(mp4_path, video_output_folder, VIDEO_FRAME_FPS):
-            log_message(f"❌ Failed to extract frames from {os.path.basename(mp4_path)}. Continuing with other videos.")
+        if extract_frames(video_path, video_output_folder, VIDEO_FRAME_FPS):
+            frames_extracted_count += 1
+        else:
+            log_message(f"❌ Failed to extract frames from {os.path.basename(video_path)}. Continuing with other videos.")
             # Clean up partially extracted frames for this video
             if os.path.exists(video_output_folder):
                 shutil.rmtree(video_output_folder)
 
     # Check if any frames were extracted for the entire course folder
-    if not os.listdir(course_video_extract_dir):
+    if frames_extracted_count == 0:
         log_message(f"⚠️ No frames extracted for any video in {course_folder_name}. Skipping zipping and upload to BG3.")
-        shutil.rmtree(course_video_extract_dir)
+        if os.path.exists(course_video_extract_dir):
+            shutil.rmtree(course_video_extract_dir)
         return False
 
     course_zip_path = os.path.join(ZIPPED_FRAMES_FOLDER, f"{course_folder_name}_frames.zip")
@@ -345,24 +350,32 @@ def extract_and_upload_rar(rar_path: str, processed_rars_set: set, uploaded_fold
     current_extract_folder = os.path.join(EXTRACT_FOLDER, f"{folder_name}_extracted")
 
     # Check if RAR is already processed (uploaded to BG2 or video frames processed)
-    if filename in processed_rars_set and (not DEST_REPO_ID_RAR or get_folder_hash(folder_name) in uploaded_folders_set) and folder_name in processed_video_courses_set:
+    # This logic needs to be careful. If BG2 is not set, we only care about video processing.
+    # If video processing is not needed, we only care about BG2 upload.
+    is_bg2_processed = (not DEST_REPO_ID_RAR) or (get_folder_hash(folder_name) in uploaded_folders_set)
+    is_bg3_processed = (folder_name in processed_video_courses_set)
+
+    if filename in processed_rars_set and is_bg2_processed and is_bg3_processed:
         log_message(f"⏩ {filename} already fully processed, skipping.")
         return True
 
-    # Generate folder name and hash for tracking
-    folder_hash = get_folder_hash(folder_name)
-    
     # If BG2 upload is enabled and folder already uploaded to BG2, skip RAR extraction/upload to BG2
-    if DEST_REPO_ID_RAR and folder_hash in uploaded_folders_set:
-        log_message(f"🔒 Folder \'{folder_name}\' already uploaded to BG2 (hash: {folder_hash[:8]}...), skipping RAR upload.")
-        processed_rars_set.add(filename)
-        save_processed_files_state(processed_rars_set)
-        # Proceed to video processing if not already done
-        if not os.path.exists(current_extract_folder): # If extracted folder doesn't exist, we can't process videos
-            log_message(f"⚠️ Extracted folder {current_extract_folder} not found for video processing. Skipping.")
-            return False
-        process_video_frames_and_upload(current_extract_folder, processed_video_courses_set)
-        return True
+    # but still proceed to video processing if not already done.
+    if DEST_REPO_ID_RAR and get_folder_hash(folder_name) in uploaded_folders_set and not is_bg3_processed:
+        log_message(f"🔒 Folder \'{folder_name}\' already uploaded to BG2 (hash: {get_folder_hash(folder_name)[:8]}...), skipping RAR upload.")
+        # If the extracted folder doesn't exist, we can't process videos from it.
+        # This scenario might happen if the previous run was interrupted after BG2 upload but before video processing cleanup.
+        if not os.path.exists(current_extract_folder): 
+            log_message(f"⚠️ Extracted folder {current_extract_folder} not found for video processing. Attempting re-extraction for video processing.")
+            # Fall through to re-extract and process videos
+        else:
+            # Proceed to video processing if not already done
+            log_message(f"Continuing with video processing for {filename}.")
+            video_processed = process_video_frames_and_upload(current_extract_folder, processed_video_courses_set)
+            if video_processed:
+                processed_rars_set.add(filename)
+                save_processed_files_state(processed_rars_set)
+            return video_processed
 
     log_message(f"📦 Attempting to extract: {filename}")
     os.makedirs(current_extract_folder, exist_ok=True)
@@ -399,31 +412,50 @@ def extract_and_upload_rar(rar_path: str, processed_rars_set: set, uploaded_fold
                     log_message(f"⬆️ Uploading to BG2: {path_in_repo}")
                     
                     try:
-                        upload_file_to_hf(
+                        if upload_file_to_hf(
                             local_path=local_path,
                             path_in_repo=path_in_repo,
                             repo_id=DEST_REPO_ID_RAR
-                        )
-                        upload_count += 1
+                        ):
+                            upload_count += 1
+                        else:
+                            log_message(f"❌ Failed to upload {path_in_repo} to BG2. Skipping remaining uploads for this RAR.")
+                            # Consider if you want to fail the whole RAR processing here or continue.
+                            # For now, we'll continue but log the failure.
+
                     except Exception as upload_error:
                         log_message(f"❌ Failed to upload {path_in_repo} to BG2: {upload_error}")
-                        raise upload_error
+                        # Don't re-raise, allow other files to be attempted
             
-            log_message(f"✅ Successfully uploaded {upload_count} files from {filename} to BG2")
-            
-            # Mark folder as uploaded to BG2 using hash
-            uploaded_folders_set.add(folder_hash)
-            save_uploaded_folders(uploaded_folders_set)
-            processing_status["uploaded_rar_folders"] = len(uploaded_folders_set)
-            
-            log_message(f"🔒 Folder \'{folder_name}\' locked in BG2 repo (hash: {folder_hash[:8]}...)")
+            if upload_count > 0: # Only mark as uploaded if at least one file was successfully uploaded
+                log_message(f"✅ Successfully uploaded {upload_count} files from {filename} to BG2")
+                # Mark folder as uploaded to BG2 using hash
+                uploaded_folders_set.add(folder_hash)
+                save_uploaded_folders(uploaded_folders_set)
+                processing_status["uploaded_rar_folders"] = len(uploaded_folders_set)
+                log_message(f"🔒 Folder \'{folder_name}\' locked in BG2 repo (hash: {folder_hash[:8]}...)")
+            else:
+                log_message(f"⚠️ No files were successfully uploaded from {filename} to BG2.")
         else:
             log_message("Skipping upload to BG2 as DEST_REPO_ID_RAR is not set.")
         
         # Now process video frames from the extracted content and upload to BG3
-        process_video_frames_and_upload(current_extract_folder, processed_video_courses_set)
+        video_processed = process_video_frames_and_upload(current_extract_folder, processed_video_courses_set)
 
-        return True
+        # Mark RAR as processed only if both BG2 (if enabled) and BG3 processing are successful
+        # Or if BG2 is not enabled, only BG3 processing needs to be successful
+        if (not DEST_REPO_ID_RAR or (folder_hash in uploaded_folders_set)) and video_processed:
+            return True
+        elif DEST_REPO_ID_RAR and not (folder_hash in uploaded_folders_set):
+            log_message(f"❌ RAR processing failed for {filename}: BG2 upload was not successful.")
+            return False
+        elif not video_processed:
+            log_message(f"❌ RAR processing failed for {filename}: Video frame processing was not successful.")
+            return False
+        else:
+            # This case should ideally not be reached if the above logic is exhaustive
+            log_message(f"❌ RAR processing failed for {filename}: Unknown reason.")
+            return False
 
     except subprocess.CalledProcessError as e:
         error_msg = f"RAR extraction failed (exit {e.returncode}): {e.stderr.strip()}"
@@ -471,49 +503,51 @@ def continuous_processing(start_download_index: Optional[int] = None):
             downloaded_rar_paths, next_download_index = download_rar_files(download_start_index, CHUNK_SIZE)
             save_download_state(next_download_index)
 
-            if not downloaded_rar_paths:
-                # Check if there are any local RAR files to process
-                all_local_rars = sorted([os.path.join(DOWNLOAD_FOLDER, f) for f in os.listdir(DOWNLOAD_FOLDER) if f.endswith(".rar")])
-                processed_rars = load_processed_files_state()
-                unprocessed_rars = [rar for rar in all_local_rars if os.path.basename(rar) not in processed_rars]
-                
-                if not unprocessed_rars:
-                    log_message("✅ No more RAR files to download. Stopping...")
-                    break
-                else:
-                    log_message(f"📋 Found {len(unprocessed_rars)} unprocessed local RAR files")
-            
             # 2. Process all available RAR files (downloaded + existing)
             all_local_rars = sorted([os.path.join(DOWNLOAD_FOLDER, f) for f in os.listdir(DOWNLOAD_FOLDER) if f.endswith(".rar")])
             processed_rars = load_processed_files_state()
             processing_status["total_files"] = len(all_local_rars)
+            # Recalculate processed_files based on actual processed_rars_set to be accurate
             processing_status["processed_files"] = len(processed_rars)
 
+            # Filter out RARs that are already fully processed based on current state
+            rars_to_process = []
             for rar_file_path in all_local_rars:
+                filename = os.path.basename(rar_file_path)
+                folder_name = filename.replace(".rar", "")
+                is_bg2_processed = (not DEST_REPO_ID_RAR) or (get_folder_hash(folder_name) in uploaded_folders)
+                is_bg3_processed = (folder_name in processed_video_courses)
+                
+                if not (filename in processed_rars and is_bg2_processed and is_bg3_processed):
+                    rars_to_process.append(rar_file_path)
+            
+            if not downloaded_rar_paths and not rars_to_process:
+                log_message("✅ No more RAR files to download or process. Stopping...")
+                break
+            
+            for rar_file_path in rars_to_process:
                 if not processing_status["is_running"]:
                     break
                     
                 filename = os.path.basename(rar_file_path)
-                if filename not in processed_rars:
-                    success = extract_and_upload_rar(rar_file_path, processed_rars, uploaded_folders, processed_video_courses)
-                    if success:
-                        processed_rars.add(filename)
-                        save_processed_files_state(processed_rars)
-                        processing_status["processed_files"] += 1
-                        
-                        # Delete the RAR file after successful processing
-                        log_message(f"🗑️ Deleting processed RAR: {filename}")
-                        try:
-                            os.remove(rar_file_path)
-                            log_message(f"✅ Deleted RAR file: {filename}")
-                        except Exception as e:
-                            log_message(f"⚠️ Could not delete {rar_file_path}: {e}")
+                success = extract_and_upload_rar(rar_file_path, processed_rars, uploaded_folders, processed_video_courses)
+                if success:
+                    # processed_rars.add(filename) is handled inside extract_and_upload_rar now for better atomicity
+                    # processing_status["processed_files"] += 1 is also handled inside
                     
-                    # Add delay between processing files
-                    time.sleep(PROCESSING_DELAY)
+                    # Delete the RAR file after successful processing
+                    log_message(f"🗑️ Deleting processed RAR: {filename}")
+                    try:
+                        os.remove(rar_file_path)
+                        log_message(f"✅ Deleted RAR file: {filename}")
+                    except Exception as e:
+                        log_message(f"⚠️ Could not delete {rar_file_path}: {e}")
+                
+                # Add delay between processing files
+                time.sleep(PROCESSING_DELAY)
 
             # If no new files were downloaded and all local files are processed, we're done
-            if not downloaded_rar_paths:
+            if not downloaded_rar_paths and not rars_to_process:
                 break
                 
     except Exception as e:
@@ -531,8 +565,8 @@ async def root():
     <html>
     <head>
         <title>RAR & Video Processing Service</title>
-        <meta charset="utf-8">
-        <meta name="viewport" content="width=device-width, initial-scale=1">
+        <meta charset=\"utf-8\">
+        <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">
         <style>
             body { font-family: Arial, sans-serif; margin: 20px; background-color: #f5f5f5; }
             .container { max-width: 1200px; margin: 0 auto; background: white; padding: 20px; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }
@@ -546,59 +580,59 @@ async def root():
             .stats { display: flex; gap: 20px; margin: 20px 0; }
             .stat-item { background: #f0f0f0; padding: 10px; border-radius: 5px; text-align: center; flex: 1; }
             .start-form { margin-top: 20px; padding: 15px; border: 1px solid #ddd; border-radius: 5px; background: #f9f9f9; }
-            .start-form input[type="number"] { width: calc(100% - 120px); padding: 8px; margin-right: 10px; border: 1px solid #ccc; border-radius: 4px; }
+            .start-form input[type=\"number\"] { width: calc(100% - 120px); padding: 8px; margin-right: 10px; border: 1px solid #ccc; border-radius: 4px; }
             .start-form button { padding: 8px 15px; background: #4CAF50; color: white; border: none; border-radius: 4px; cursor: pointer; }
             .start-form button:hover { background: #45a049; }
         </style>
     </head>
     <body>
-        <div class="container">
+        <div class=\"container\">
             <h1>🔄 RAR & Video Processing Service</h1>
             <p>Automated extraction and upload of RAR files from BG1 to BG2 dataset, and video frame extraction/upload to BG3 dataset</p>
             
-            <div class="status-card">
-                <h3>Status: <span id="status">Stopped</span></h3>
-                <p>Current File: <span id="current-file">None</span></p>
-                <p>Last Update: <span id="last-update">Never</span></p>
+            <div class=\"status-card\">
+                <h3>Status: <span id=\"status\">Stopped</span></h3>
+                <p>Current File: <span id=\"current-file\">None</span></p>
+                <p>Last Update: <span id=\"last-update\">Never</span></p>
             </div>
             
-            <div class="stats">
-                <div class="stat-item">
+            <div class=\"stats\">
+                <div class=\"stat-item\">
                     <h4>Total Files (RARs)</h4>
-                    <span id="total-files">0</span>
+                    <span id=\"total-files\">0</span>
                 </div>
-                <div class="stat-item">
+                <div class=\"stat-item\">
                     <h4>Processed (RARs)</h4>
-                    <span id="processed-files">0</span>
+                    <span id=\"processed-files\">0</span>
                 </div>
-                <div class="stat-item">
+                <div class=\"stat-item\">
                     <h4>Uploaded Folders (BG2)</h4>
-                    <span id="uploaded-rar-folders">0</span>
+                    <span id=\"uploaded-rar-folders\">0</span>
                 </div>
-                <div class="stat-item">
+                <div class=\"stat-item\">
                     <h4>Uploaded Video Courses (BG3)</h4>
-                    <span id="uploaded-video-courses">0</span>
+                    <span id=\"uploaded-video-courses\">0</span>
                 </div>
-                <div class="stat-item">
+                <div class=\"stat-item\">
                     <h4>Failed</h4>
-                    <span id="failed-files">0</span>
+                    <span id=\"failed-files\">0</span>
                 </div>
             </div>
             
-            <div class="start-form">
+            <div class=\"start-form\">
                 <h3>Start Processing from Specific Download Index</h3>
-                <input type="number" id="start-index-input" placeholder="Enter start index (e.g., 0)" value="0">
-                <button onclick="startProcessingWithIndex()">Start from Index</button>
+                <input type=\"number\" id=\"start-index-input\" placeholder=\"Enter start index (e.g., 0)\" value=\"0\">
+                <button onclick=\"startProcessingWithIndex()\">Start from Index</button>
             </div>
 
             <div>
-                <button class="button" onclick="startProcessing()" id="start-btn">Start Processing (from last saved index)</button>
-                <button class="button stop-button" onclick="stopProcessing()" id="stop-btn" disabled>Stop Processing</button>
-                <button class="button" onclick="refreshStatus()">Refresh Status</button>
+                <button class=\"button\" onclick=\"startProcessing()\" id=\"start-btn\">Start Processing (from last saved index)</button>
+                <button class=\"button stop-button\" onclick=\"stopProcessing()\" id=\"stop-btn\" disabled>Stop Processing</button>
+                <button class=\"button\" onclick=\"refreshStatus()\">Refresh Status</button>
             </div>
             
             <h3>Logs</h3>
-            <div class="logs" id="logs">Loading...</div>
+            <div class=\"logs\" id=\"logs\">Loading...</div>
         </div>
         
         <script>
@@ -615,7 +649,7 @@ async def root():
 
             async function startProcessingWithIndex() {
                 const index = document.getElementById(\"start-index-input\").value;
-                if (index === "" || isNaN(index)) {
+                if (index === \"\" || isNaN(index)) {
                     alert(\"Please enter a valid number for the start index.\");
                     return;
                 }
@@ -733,4 +767,6 @@ async def get_processed_video_courses():
     return {"processed_video_course_count": len(processed_video_courses), "course_names": list(processed_video_courses)}
 
 if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=7860)
+    uvicorn.run(app, host="0.0.0.0", port=7860)
+
+