Update app.py

app.py

@@ -1,3 +1,4 @@
+
 import os
 import re
 import time
@@ -116,7 +117,7 @@ def list_rar_files_in_dataset() -> list:
         rar_files = [
             f for f in all_files 
             if f.startswith(DATA_PATH) 
-            and (f.lower().endswith('.rar') or re.search(r'\.r\d{2}$', f, re.IGNORECASE))
+            and (f.lower().endswith(".rar") or re.search(r'\.r\d{2}$', f, re.IGNORECASE))
         ]
         
         logging.info(f"[*] Found {len(rar_files)} RAR files")
@@ -128,6 +129,32 @@ def list_rar_files_in_dataset() -> list:
         logging.error(f"[!] Failed to list files: {e}")
         return []
 
+# === List Extracted Courses in Dataset ===
+def list_extracted_courses_in_dataset() -> set:
+    try:
+        logging.info(f"[*] Scanning dataset for extracted courses in {EXTRACTED_PATH}")
+        all_files = list_repo_files(
+            repo_id=REPO_ID,
+            repo_type="dataset",
+            token=HF_TOKEN
+        )
+        
+        extracted_courses = set()
+        for f in all_files:
+            if f.startswith(EXTRACTED_PATH + '/') and len(f.split('/')) > len(EXTRACTED_PATH.split('/')):
+                # Extract the course name from the path
+                course_name = f.split('/')[len(EXTRACTED_PATH.split('/'))]
+                extracted_courses.add(course_name)
+        
+        logging.info(f"[*] Found {len(extracted_courses)} extracted courses")
+        for ec in extracted_courses:
+            logging.info(f"    - {ec}")
+            
+        return extracted_courses
+    except Exception as e:
+        logging.error(f"[!] Failed to list extracted courses: {e}")
+        return set()
+
 # === Extract and Upload RAR ===
 async def extract_and_upload_rar(rar_file_path: str):
     try:
@@ -173,13 +200,22 @@ async def rar_processor_worker():
         try:
             logging.info("[*] Scanning for RAR files...")
             rar_files = list_rar_files_in_dataset()
+            extracted_courses = list_extracted_courses_in_dataset()
+            
+            untouched_rar_files = []
+            for rar_file in rar_files:
+                course_name = extract_course_name(os.path.basename(rar_file))
+                if course_name not in extracted_courses:
+                    untouched_rar_files.append(rar_file)
             
-            if rar_files:
-                for rar_file in rar_files:
+            if untouched_rar_files:
+                logging.info(f"[*] Found {len(untouched_rar_files)} untouched RAR files.")
+                for rar_file in untouched_rar_files:
+                    logging.info(f"    - {rar_file}")
                     await extract_and_upload_rar(rar_file)
                     await asyncio.sleep(5)  # Delay between files
             else:
-                logging.info("[*] No RAR files found, waiting...")
+                logging.info("[*] No untouched RAR files found, waiting...")
             
             # Wait 60 seconds before next scan
             await asyncio.sleep(60)
@@ -210,20 +246,35 @@ def health():
 
 @app.get("/scan")
 def scan_rars():
-    """Manually trigger RAR file scan"""
+    """Manually trigger RAR file scan and identify untouched ones"""
     rar_files = list_rar_files_in_dataset()
-    return {"found": len(rar_files), "files": rar_files}
+    extracted_courses = list_extracted_courses_in_dataset()
+    
+    untouched_rar_files = []
+    for rar_file in rar_files:
+        course_name = extract_course_name(os.path.basename(rar_file))
+        if course_name not in extracted_courses:
+            untouched_rar_files.append(rar_file)
+            
+    return {"total_rar_files": len(rar_files), "extracted_courses": len(extracted_courses), "untouched_rar_files": untouched_rar_files}
 
 @app.post("/extract-all")
 async def extract_all():
-    """Manually trigger extraction of all RAR files"""
+    """Manually trigger extraction of all untouched RAR files"""
     rar_files = list_rar_files_in_dataset()
+    extracted_courses = list_extracted_courses_in_dataset()
     
-    if not rar_files:
-        return {"message": "No RAR files found"}
+    untouched_rar_files = []
+    for rar_file in rar_files:
+        course_name = extract_course_name(os.path.basename(rar_file))
+        if course_name not in extracted_courses:
+            untouched_rar_files.append(rar_file)
+
+    if not untouched_rar_files:
+        return {"message": "No untouched RAR files found to extract"}
     
     results = []
-    for rar_file in rar_files:
+    for rar_file in untouched_rar_files:
         success = await extract_and_upload_rar(rar_file)
         results.append({"file": rar_file, "success": success})
         await asyncio.sleep(5)