Create app.py

app.py

@@ -0,0 +1,99 @@
+import os
+import time
+import requests
+from urllib.parse import urlparse
+from huggingface_hub import upload_file
+from fastapi import FastAPI
+from contextlib import asynccontextmanager
+import asyncio
+import logging
+
+# === CONFIGURATION ===
+HF_TOKEN = os.environ.get("HF_TOKEN")
+REPO_ID = "Fred808/BG1"
+DATA_PATH = "Huidini"
+OUTPUT_DIR = "batch_downloads"
+DOWNLOAD_URLS = [
+    "https://ww8.zeroupload.xyz/025984a8fb2726d8e39b334d23c79360/MDA_TheSymbioteCreature_DownloadPirate.com.rar?download_token=2912b71a4669baaec51b23d28a1665d6ebadedb5f7cf89706a8111302838ca15",
+    "https://ww8.zeroupload.xyz/4f2f3f75fde2434d5946170303d08196/MDA_MasteringHoudiniArtDesign_DownloadPirate.com.rar?download_token=f2e041102d5c54df37949603fa6a8fe84ffb1611587bcf8f5f9cfa801454f9ed",   
+]
+DELAY_BETWEEN_DOWNLOADS = 12  # seconds
+
+# === Setup Logging ===
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+
+# === Prepare output folder ===
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+app = FastAPI()
+
+# === DUMMY ROUTE TO KEEP SERVER HEALTHY ===
+@app.get("/")
+def keep_alive():
+    return {"status": "running"}
+
+# === Upload Function ===
+def upload_to_dataset(filepath):
+    try:
+        upload_file(
+            path_or_fileobj=filepath,
+            path_in_repo=f"{DATA_PATH}/{os.path.basename(filepath)}",
+            repo_id=REPO_ID,
+            repo_type="dataset",
+            token=HF_TOKEN
+        )
+        logging.info(f"[↑] Uploaded: {filepath}")
+    except Exception as e:
+        logging.error(f"[!] Upload failed: {filepath} — {e}")
+
+# === Background Worker ===
+async def downloader_worker():
+    for direct_download_link in DOWNLOAD_URLS:
+        logging.info("[*] Waiting before next download...")
+        await asyncio.sleep(DELAY_BETWEEN_DOWNLOADS)
+
+        try:
+            logging.info(f"[*] Downloading from: {direct_download_link}")
+            filename = os.path.basename(urlparse(direct_download_link).path)
+            if not filename or "." not in filename:
+                filename = "downloaded_file_" + str(int(time.time()))
+
+            local_path = os.path.join(OUTPUT_DIR, filename)
+            logging.info(f"[*] Saving to: {local_path}")
+
+            with requests.get(direct_download_link, stream=True) as r:
+                r.raise_for_status()
+                with open(local_path, "wb") as f:
+                    for chunk in r.iter_content(chunk_size=8192):
+                        f.write(chunk)
+
+            logging.info(f"[✓] Downloaded: {filename}")
+            upload_to_dataset(local_path)
+            os.remove(local_path)
+
+        except Exception as e:
+            logging.error(f"[!] Error with {direct_download_link}: {e}")
+
+    logging.info("✅ All files processed.")
+
+@app.get("/")
+def stay_alive():
+    return {"msg": "Running"}
+
+@app.get("/health")
+def healthcheck():
+    return {"healthy": True}
+
+# === FastAPI Lifespan ===
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    logging.info("🚀 Starting FastAPI download-uploader microservice...")
+    task = asyncio.create_task(downloader_worker())
+    yield
+    task.cancel()
+    logging.info("🛑 Shutting down microservice.")
+
+# === FastAPI App ===
+app = FastAPI(lifespan=lifespan)
+# Re-assign app with lifespan logic
+app.router.lifespan_context = lifespan