Update app.py

app.py

@@ -68,7 +68,8 @@ VIDEO_URLS = [
     "https://youtu.be/XGHBtvnvz9U"
 ]
 
-# === Load processed videos ===
+
+# === Load processed ===
 if os.path.exists(PROCESSED_FILE):
     with open(PROCESSED_FILE, "r") as f:
         processed_urls = set(json.load(f))
@@ -93,21 +94,24 @@ def upload_to_dataset(filepath):
             token=HF_TOKEN
         )
         print(f"[↑] Uploaded: {filepath}")
+        return True
     except Exception as e:
         print(f"[!] Upload failed: {filepath} — {e}")
+        return False
 
-# === Create output folder ===
+# === Prepare output folder ===
 if os.path.exists(OUTPUT_DIR):
-    for file in os.listdir(OUTPUT_DIR):
-        os.remove(os.path.join(OUTPUT_DIR, file))
+    for f in os.listdir(OUTPUT_DIR):
+        os.remove(os.path.join(OUTPUT_DIR, f))
 else:
     os.makedirs(OUTPUT_DIR)
 
-# === Process in batches ===
+# === Filter unprocessed URLs ===
 unprocessed_urls = [url for url in VIDEO_URLS if url not in processed_urls]
 
+# === Process in batches ===
 for batch in chunk_urls(unprocessed_urls, 20):
-    print(f"\n[*] Submitting {len(batch)} URLs to batch API...")
+    print(f"\n[*] Submitting batch of {len(batch)} URLs...")
     try:
         res = requests.post(INIT_API_URL, json={"urls": batch})
         res.raise_for_status()
@@ -119,39 +123,32 @@ for batch in chunk_urls(unprocessed_urls, 20):
         print(f"[!] Submission error: {e}")
         continue
 
-    # === Wait for batch to finish ===
+    # === Wait until all videos in batch are fully processed ===
     status_url = STATUS_API_URL.format(batch_id)
-    print("[*] Waiting for batch to complete...")
-    POLL_DELAY = 5
-    MAX_WAIT_MINUTES = 5
-    MAX_RETRIES = int((MAX_WAIT_MINUTES * 60) / POLL_DELAY)
-
-    for attempt in range(MAX_RETRIES):
+    print("[*] Waiting for all batch videos to be processed...")
+    while True:
         try:
             res = requests.get(status_url)
             res.raise_for_status()
             data = res.json()
-            status = data.get("status")
+
             total = data.get("total_urls", "?")
             completed = data.get("completed", 0)
             failed = data.get("failed", 0)
+            status = data.get("status")
 
-            if status == "completed":
-                print(f"[✓] All {completed}/{total} videos processed.")
-                break
-            elif status in ["started", "processing", "in_progress"]:
-                print(f"[~] Processing... {completed}/{total} done, {failed} failed.")
-            elif status == "failed":
-                print("[!] Batch failed on server.")
+            print(f"[~] Status: {status} — {completed}/{total} done, {failed} failed.")
+
+            if completed + failed >= int(total):
+                print(f"[✓] Batch fully processed: {completed} completed, {failed} failed.")
                 break
+
         except Exception as e:
-            print(f"[!] Status check failed: {e}")
-        time.sleep(POLL_DELAY)
-    else:
-        print("[!] Timeout — skipping batch.")
-        continue
+            print(f"[!] Error checking status: {e}")
 
-    # === Fetch download links ===
+        time.sleep(5)
+
+    # === Fetch download list ===
     fetch_url = FETCH_API_URL.format(batch_id)
     print("[*] Fetching download list...")
     try:
@@ -166,7 +163,7 @@ for batch in chunk_urls(unprocessed_urls, 20):
         print(f"[!] Failed to fetch download list: {e}")
         continue
 
-    # === Download and upload ===
+    # === Download & upload ===
     for video, url in zip(downloads, batch):
         filename = video.get("filename")
         file_url = video.get("url")
@@ -178,6 +175,7 @@ for batch in chunk_urls(unprocessed_urls, 20):
             file_url = BASE_URL + file_url
 
         local_path = os.path.join(OUTPUT_DIR, filename)
+
         try:
             print(f"[*] Downloading {filename}...")
             with requests.get(file_url, stream=True) as r:
@@ -186,14 +184,17 @@ for batch in chunk_urls(unprocessed_urls, 20):
                     for chunk in r.iter_content(chunk_size=8192):
                         f.write(chunk)
             print(f"[✓] Downloaded: {filename}")
-            upload_to_dataset(local_path)
+        except Exception as e:
+            print(f"[!] Download failed: {filename} — {e}")
+            continue
+
+        # Upload
+        if upload_to_dataset(local_path):
             processed_urls.add(url)
             save_processed()
-            os.remove(local_path)
-        except Exception as e:
-            print(f"[!] Error downloading/uploading {filename}: {e}")
+        os.remove(local_path)
 
     print("[⏱] Waiting 30s before next batch...\n")
     time.sleep(30)
 
-print("\n✅ Done. All batches processed.")
+print("\n✅ Done. All batches fully processed.")