Update app.py

app.py

@@ -1,104 +1,20 @@
 import os
 import time
 import json
-import requests
+import asyncio
+from torrentp import TorrentDownloader
 from huggingface_hub import upload_file
 
 # === CONFIGURATION ===
 HF_TOKEN = os.environ.get("HF_TOKEN")
 REPO_ID = "Fred808/BG1"
 DATA_PATH = "AEffects"
-BASE_URL = "https://fred808-yt1.hf.space"
-INIT_API_URL = f"{BASE_URL}/batch/download"
-STATUS_API_URL = f"{BASE_URL}/batch/status/{{}}"
-FETCH_API_URL = f"{BASE_URL}/batch/download-all/{{}}"
 PROCESSED_FILE = "processed.json"
 OUTPUT_DIR = "batch_downloads"
 
-VIDEO_URLS = [
- "https://youtu.be/__gJ9VL9z-c",
- "https://youtu.be/bAcmb9PGzIE",
- "https://youtu.be/aO0eUnu0hO0",
- "https://youtu.be/fsLO1F5x7yM",
- "https://youtu.be/XcJyW5iBKlU",
- "https://youtu.be/md8mbgTEfIk",
- "https://www.youtube.com/playlist?list=PLxj9fixyu7Pbhy8o_Teys78waXmoxmQVZ",
- "https://www.youtube.com/playlist?list=PLsGl9GczcgBsv58A4sTYSmLDhipykRS4O",
- "https://youtu.be/Wg244y2f9Fw",
- "https://m.youtube.com/watch?v=aafyp5g1CjI",
- "https://youtu.be/Aj1x89EFc8M",
- "https://youtu.be/SNHBFtdGWhg",
- "https://www.youtube.com/playlist?list=PLVm7O9OzjT6EIDtFGC67QxGsHkz3_RbIw",
- "https://www.youtube.com/playlist?list=PL-J0k20j-Ffh5IWHjeT2Je-asx7YkFbj6",
- "https://www.youtube.com/playlist?list=PLsGl9GczcgBsZdwrobBkXT_A8jse09yRz",
- "https://www.youtube.com/c/BlenderMadeEasy",
- "https://youtu.be/8tViVXgu7fo",
- "https://youtu.be/aTeLyeVO0Dc",
- "https://youtu.be/RRilLLyyn1Y",
- "https://youtu.be/567FFSk23Io",
- "https://youtu.be/KO29y5eW61g",
- "https://youtu.be/Y1bOokHzEAk",
- "https://youtu.be/k-mEz54_p90",
- "https://www.youtube.com/playlist?list=PLNShHVjao84dTJDUQA9dPNTGsox16p1iV",
- "https://youtu.be/tWvgHbZXCtA",
- "https://youtu.be/4yrsAiTdMj4",
- "https://youtu.be/HDBZ7CwlfIs",
- "https://youtu.be/vuiS9Ypanu8",
- "https://youtu.be/-F6oaJzyr4A",
- "https://youtu.be/nxU4jXUYPEg",
- "https://youtu.be/abafsArInDE",
- "https://youtu.be/v0pIfh3Unz0",
- "https://youtu.be/Td8GUnhZeao",
- "https://youtu.be/VlSs430PAeU",
- "https://youtu.be/xt5dWu15uM4",
- "https://youtu.be/c3FnWQTMo9s",
- "https://youtu.be/B0J27sf9N1Y",
- "https://youtu.be/lLqep5Q4MiI",
- "https://youtu.be/Q30-nakUrSM",
- "https://youtu.be/HSm-cq7zd2s",
- "https://youtu.be/x6oWgtJInCQ",
- "https://youtu.be/9gn_1V1sCS8",
- "https://youtu.be/dIv2FXyD3CU",
- "https://youtu.be/ABayYXu7OfI",
- "https://youtu.be/-c0Evpf8V3A",
- "https://www.youtube.com/playlist?list=PLgO2ChD7acqHzccBuhAGw8dTPLnR1E3QB",
- "https://www.youtube.com/user/AndrewPPrice/videos",
- "https://www.youtube.com/playlist?list=PL2aDImegRwZFMIdrSdn2DCapf98I-LJsf",
- "https://youtu.be/n6V8v5PlvGI",
- "https://youtu.be/XO783U-B5bg",
- "https://youtu.be/G17sBkp-DIk",
- "https://youtu.be/VMLW6XW0k6U",
- "https://youtu.be/WgZ1K3J3RGU",
- "https://youtu.be/DEx1nwxRzXQ",
- "https://youtu.be/xbKDdRDLhJ8",
- "https://youtu.be/F5Z8rj-fekU",
- "https://youtu.be/wkX0z6ygng4",
- "https://youtu.be/rg-VJGf3Z8E",
- "https://youtu.be/Bkme3OeK6DM",
- "https://youtu.be/FsniCv0L-7E",
- "https://youtu.be/fZrY5n-wqZQ",
- "https://youtu.be/XGHBtvnvz9U",
- "https://youtu.be/SXOXeN5uzVw",
- "https://youtu.be/Pv8P8yYvPMY",
- "https://youtu.be/J5t7gzIfTt0",
- "https://youtu.be/xtdE6TLkOkQ",
- "https://youtu.be/AY1GBG8PBuU",
- "https://youtu.be/R1tv1mgiOkQ",
- "https://youtu.be/HJSGo",
- "https://youtu.be/5B244CYX1Tw",
- "https://youtu.be/OPCWsTtIZ8Y",
- "https://youtu.be/FxLy1X3DLhw",
- "https://youtu.be/PSptnmU1GRg",
- "https://youtu.be/S4jF-8zKRYA",
- "https://youtu.be/N_F5hpeiPJA",
- "https://youtu.be/7ZJ2WuqjX9w",
- "https://youtu.be/F8kCZwJg3VA",
- "https://youtu.be/tbJY_k1jM7w",
- "https://youtu.be/kDwS1zqD4XY",
- "https://youtu.be/mK9B1E4wKZI",
- "https://youtu.be/QY4EVmF3_mM",
- "https://youtu.be/A2P9BZez8L0",
- "https://youtu.be/U7Qxp5-VWXs"
+# Placeholder for magnet links - these will replace the original VIDEO_URLS
+MAGNET_LINKS = [
+    "magnet:?xt=urn:btih:18ED0A9276EA0324DFE1683A13E52DEB7C36CC1B&dn=Digital+Tutors+Creating+A+High-Speed+Camera+Effect+With+After+Effects&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.publicbt.com%3A80%2Fannounce&tr=udp%3A%2F%2Ftracker.prq.to%2Fannounce&tr=udp%3A%2F%2F10.rarbg.me%3A80%2Fannounce&tr=udp%3A%2F%2Ftracker.token.ro%3A80%2Fannounce&tr=udp%3A%2F%2F10.rarbg.com%2Fannounce&tr=http%3A%2F%2Ftracker.torrentfrancais.com%2Fannounce&tr=udp%3A%2F%2F9.rarbg.com%3A2710%2Fannounce&tr=udp%3A%2F%2Ftracker.openbittorrent.com%3A80%2Fannounce&tr=udp%3A%2F%2F11.rarbg.com%2Fannounce&tr=http%3A%2F%2Ftracker.ex.ua%2Fannounce&tr=http%3A%2F%2Ftorrent.gresille.org%2Fannounce&tr=http%3A%2F%2F94.228.192.98%2Fannounce&tr=udp%3A%2F%2Ftracker.istole.it%3A80%2Fannounce&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=http%3A%2F%2Ftracker.openbittorrent.com%3A80%2Fannounce&tr=udp%3A%2F%2Fopentracker.i2p.rocks%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.internetwarriors.net%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969%2Fannounce&tr=udp%3A%2F%2Fcoppersurfer.tk%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.zer0day.to%3A1337%2Fannounce"
 ]
 
 # === Load processed videos ===
@@ -112,10 +28,6 @@ def save_processed():
     with open(PROCESSED_FILE, "w") as f:
         json.dump(list(processed_urls), f, indent=2)
 
-def chunk_urls(lst, n):
-    for i in range(0, len(lst), n):
-        yield lst[i:i + n]
-
 def upload_to_dataset(filepath):
     try:
         upload_file(
@@ -136,94 +48,47 @@ if os.path.exists(OUTPUT_DIR):
 else:
     os.makedirs(OUTPUT_DIR)
 
-# === Process in batches ===
-unprocessed_urls = [url for url in VIDEO_URLS if url not in processed_urls]
-
-for batch in chunk_urls(unprocessed_urls, 20):
-    print(f"\n[*] Submitting batch of {len(batch)} URLs...")
+async def download_and_upload_torrent(magnet_link):
+    print(f"[*] Starting download for: {magnet_link}")
     try:
-        res = requests.post(INIT_API_URL, json={"urls": batch})
-        res.raise_for_status()
-        batch_id = res.json().get("batch_id")
-        if not batch_id:
-            raise Exception("No batch_id returned.")
-        print(f"[+] Batch submitted. ID: {batch_id}")
-    except Exception as e:
-        print(f"[!] Submission error: {e}")
-        continue
-
-    # === Wait until all videos in batch are fully processed ===
-    status_url = STATUS_API_URL.format(batch_id)
-    print("[*] Waiting for all batch videos to be processed...")
-
-    while True:
-        try:
-            res = requests.get(status_url)
-            res.raise_for_status()
-            data = res.json()
-
-            total = int(data.get("total_urls", 0))
-            completed = data.get("completed", 0)
-            failed = data.get("failed", 0)
+        # Initialize TorrentDownloader with magnet link and output directory
+        # stop_after_download=True ensures it stops seeding after download
+        torrent_downloader = TorrentDownloader(magnet_link, OUTPUT_DIR, stop_after_download=True)
+        await torrent_downloader.start_download()
+        print(f"[✓] Downloaded: {magnet_link}")
+
+        # Find the downloaded file(s) in the output directory
+        # This part might need refinement based on how torrentp names files/folders
+        downloaded_files = []
+        for root, _, files in os.walk(OUTPUT_DIR):
+            for file in files:
+                downloaded_files.append(os.path.join(root, file))
+        
+        if not downloaded_files:
+            print(f"[!] No files found after downloading {magnet_link}")
+            return
+
+        for local_path in downloaded_files:
+            upload_to_dataset(local_path)
+            os.remove(local_path) # Clean up local file after upload
+        
+        processed_urls.add(magnet_link)
+        save_processed()
 
-            print(f"[~] Status: {data.get('status')} — {completed}/{total} done, {failed} failed.")
+    except Exception as e:
+        print(f"[!] Error downloading/uploading {magnet_link}: {e}")
 
-            if completed >= total:
-                print(f"[✓] Batch fully processed: {completed}/{total} completed.")
-                break
-            elif failed > 0 and completed + failed >= total:
-                print(f"[!] Batch has failed videos — retrying in 30s...")
-            else:
-                print(f"[~] Still waiting...")
+async def main():
+    unprocessed_links = [link for link in MAGNET_LINKS if link not in processed_urls]
 
-        except Exception as e:
-            print(f"[!] Error checking status: {e}")
+    for link in unprocessed_links:
+        await download_and_upload_torrent(link)
+        print("[⏱] Waiting 5s before next torrent...\n")
+        time.sleep(5) # Small delay between torrents
 
-        time.sleep(5)
+    print("\n✅ Done. All torrents processed.")
 
-    # === Fetch download links ===
-    fetch_url = FETCH_API_URL.format(batch_id)
-    print("[*] Fetching download list...")
-    try:
-        res = requests.get(fetch_url)
-        res.raise_for_status()
-        downloads = res.json().get("downloads", [])
-        if not downloads:
-            print("[!] No downloads returned.")
-            continue
-        print(f"[+] Found {len(downloads)} videos.")
-    except Exception as e:
-        print(f"[!] Failed to fetch download list: {e}")
-        continue
-
-    # === Download and upload ===
-    for video, url in zip(downloads, batch):
-        filename = video.get("filename")
-        file_url = video.get("url")
-        if not filename or not file_url:
-            print("[!] Skipping invalid entry.")
-            continue
-
-        if file_url.startswith("/"):
-            file_url = BASE_URL + file_url
-
-        local_path = os.path.join(OUTPUT_DIR, filename)
-        try:
-            print(f"[*] Downloading {filename}...")
-            with requests.get(file_url, stream=True) as r:
-                r.raise_for_status()
-                with open(local_path, "wb") as f:
-                    for chunk in r.iter_content(chunk_size=8192):
-                        f.write(chunk)
-            print(f"[✓] Downloaded: {filename}")
-            upload_to_dataset(local_path)
-            processed_urls.add(url)
-            save_processed()
-            os.remove(local_path)
-        except Exception as e:
-            print(f"[!] Error downloading/uploading {filename}: {e}")
+if __name__ == "__main__":
+    asyncio.run(main())
 
-    print("[⏱] Waiting 30s before next batch...\n")
-    time.sleep(30)
 
-print("\n✅ Done. All batches fully processed.")