Update main.py

main.py

@@ -1,35 +1,24 @@
 import os
 import time
-import json
 import requests
 from urllib.parse import urlparse
+from bs4 import BeautifulSoup
 from huggingface_hub import upload_file
 
 # === CONFIGURATION ===
 HF_TOKEN = os.environ.get("HF_TOKEN")  # Set this via environment or replace with string
-ZEROUPLOAD_TOKEN = "vcIXZnfFfZlPg5GEw4JqeqZTroJh4nst0WqFvun37SNr1wtzwsu6wC9pZ4x8ze4z"
 REPO_ID = "Fred808/BG1"
 DATA_PATH = "AEffects"
-PROCESSED_FILE = "processed.json"
 OUTPUT_DIR = "batch_downloads"
+
+# Use the direct download URL provided by the user
 DOWNLOAD_URLS = [
-   "https://zeroupload.com/e7a5f338b6db8987f1735378ad164c79"
+   "https://ww7.zeroupload.xyz/4642fd002fd25380de2ec0ea980c8bbe/After_Effects_Masterclass_Unleash_Your_Creative_Power!.zip?download_token=53f9ef43d6ad9f6eb9ed63d82b907919a99125e2116abff7d3266cdf8e12a0ee",
+    "https://ww3.zeroupload.xyz/e7a5f338b6db8987f1735378ad164c79/CGCookie_-_Blender_4.2_Core_Essentials_-_9_Tutorials.7z.003?download_token=1adf44bc722babae998608712c4704e6f7a747e641a9e09d913e2e992635ad9f"
 ]
 
-HEADERS = {
-    "Authorization": f"Bearer {ZEROUPLOAD_TOKEN}"
-}
-
-# === Load processed list ===
-if os.path.exists(PROCESSED_FILE):
-    with open(PROCESSED_FILE, "r") as f:
-        processed_urls = set(json.load(f))
-else:
-    processed_urls = set()
-
-def save_processed():
-    with open(PROCESSED_FILE, "w") as f:
-        json.dump(list(processed_urls), f, indent=2)
+# === Prepare output folder ===
+os.makedirs(OUTPUT_DIR, exist_ok=True)
 
 def upload_to_dataset(filepath):
     try:
@@ -44,52 +33,34 @@ def upload_to_dataset(filepath):
     except Exception as e:
         print(f"[!] Upload failed: {filepath} — {e}")
 
-# === Prepare output folder ===
-os.makedirs(OUTPUT_DIR, exist_ok=True)
-
 # === Start Download Loop ===
-for url in DOWNLOAD_URLS:
-    if url in processed_urls:
-        print(f"[✔] Already processed: {url}")
-        continue
-
+for direct_download_link in DOWNLOAD_URLS:
     print(f"\n[*] Waiting 12 seconds before next download...")
     time.sleep(12)
 
     try:
-        print(f"[*] Downloading from: {url}")
-        with requests.get(url, headers=HEADERS, stream=True) as r:
-            r.raise_for_status()
-
-            content_type = r.headers.get("Content-Type", "").lower()
-            if "html" in content_type:
-                raise Exception(f"[!] HTML page received instead of a file. Content-Type: {content_type}")
+        print(f"[*] Downloading from: {direct_download_link}")
 
-            # Extract filename
-            cd = r.headers.get("Content-Disposition", "")
-            if "filename=" in cd:
-                filename = cd.split("filename=")[1].strip().strip('"')
-            else:
-                # Fallback to last part of the URL
-                filename = os.path.basename(urlparse(url).path)
-                if '.' not in filename:
-                    filename += ".bin"
+        # Download the file from the direct link
+        filename = os.path.basename(urlparse(direct_download_link).path)
+        if not filename or "." not in filename: # Basic check for valid filename
+            filename = "downloaded_file" + str(int(time.time())) # Generate a unique name
 
-            local_path = os.path.join(OUTPUT_DIR, filename)
+        local_path = os.path.join(OUTPUT_DIR, filename)
+        print(f"[*] Downloading file to: {local_path}")
 
-            # Download file
+        with requests.get(direct_download_link, stream=True) as r:
+            r.raise_for_status()
             with open(local_path, "wb") as f:
                 for chunk in r.iter_content(chunk_size=8192):
                     f.write(chunk)
 
         print(f"[✓] Downloaded: {filename}")
-
         upload_to_dataset(local_path)
-        processed_urls.add(url)
-        save_processed()
         os.remove(local_path)
 
     except Exception as e:
-        print(f"[!] Error with {url}: {e}")
+        print(f"[!] Error with {direct_download_link}: {e}")
+
+print("\n✅ All files processed.")
 
-print("\n✅ All files processed.")