Upload 3 files

Dockerfile

@@ -0,0 +1,32 @@
+# Use official Python base image
+FROM python:3.10-slim
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+# Create app directory
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements and install Python packages
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the code
+COPY . .
+
+# Make everything in /app world-writable
+RUN chmod -R 777 /app
+
+# Ensure /tmp is writable too (common scratch space)
+RUN chmod -R 777 /tmp
+
+# Default command to run the script (change as needed)
+EXPOSE 8000
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

main.py

@@ -0,0 +1,142 @@
+import os
+import time
+import requests
+import zipfile
+import shutil
+from urllib.parse import urlparse
+from huggingface_hub import upload_file
+from fastapi import FastAPI
+from contextlib import asynccontextmanager
+import asyncio
+import logging
+
+# === CONFIGURATION ===
+HF_TOKEN = os.environ.get("HF_TOKEN")
+REPO_ID = "factorstudios/Pipeline"
+DATA_PATH = "Blenders"
+OUTPUT_DIR = "batch_downloads"
+DOWNLOAD_URLS = [
+   "https://ww6.zeroupload.xyz/a852d5fdd63835ba196a48462b2759af/Coloso_BlChScul_DownloadPirate.com.rar?download_token=ca9428d28e5ea75bf0a412ea2b50ce3410d2df8fdaf9073b555704f4b42ba98f"    
+]
+DELAY_BETWEEN_DOWNLOADS = 12  # seconds
+
+# === Setup Logging ===
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+
+# === Prepare output folder ===
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+app = FastAPI()
+
+# === DUMMY ROUTE TO KEEP SERVER HEALTHY ===
+@app.get("/")
+def keep_alive():
+    return {"status": "running"}
+
+# === Upload Function ===
+def upload_to_dataset(filepath):
+    try:
+        upload_file(
+            path_or_fileobj=filepath,
+            path_in_repo=f"{DATA_PATH}/{os.path.basename(filepath)}",
+            repo_id=REPO_ID,
+            repo_type="dataset",
+            token=HF_TOKEN
+        )
+        logging.info(f"[↑] Uploaded: {filepath}")
+    except Exception as e:
+        logging.error(f"[!] Upload failed: {filepath} — {e}")
+
+# === Upload Directory Contents ===
+def upload_directory_contents(directory):
+    try:
+        for root, dirs, files in os.walk(directory):
+            for file in files:
+                filepath = os.path.join(root, file)
+                relative_path = os.path.relpath(filepath, directory)
+                upload_file(
+                    path_or_fileobj=filepath,
+                    path_in_repo=f"{DATA_PATH}/{relative_path}",
+                    repo_id=REPO_ID,
+                    repo_type="dataset",
+                    token=HF_TOKEN
+                )
+                logging.info(f"[↑] Uploaded: {relative_path}")
+    except Exception as e:
+        logging.error(f"[!] Upload directory failed: {directory} — {e}")
+
+# === Background Worker ===
+async def downloader_worker():
+    for direct_download_link in DOWNLOAD_URLS:
+        logging.info("[*] Waiting before next download...")
+        await asyncio.sleep(DELAY_BETWEEN_DOWNLOADS)
+
+        try:
+            logging.info(f"[*] Downloading from: {direct_download_link}")
+            filename = os.path.basename(urlparse(direct_download_link).path)
+            if not filename or "." not in filename:
+                filename = "downloaded_file_" + str(int(time.time()))
+
+            local_path = os.path.join(OUTPUT_DIR, filename)
+            logging.info(f"[*] Saving to: {local_path}")
+
+            with requests.get(direct_download_link, stream=True) as r:
+                r.raise_for_status()
+                with open(local_path, "wb") as f:
+                    for chunk in r.iter_content(chunk_size=8192):
+                        f.write(chunk)
+
+            logging.info(f"[✓] Downloaded: {filename}")
+            
+            # Check if file is a zip file
+            if filename.lower().endswith('.zip'):
+                logging.info(f"[*] Extracting zip file: {filename}")
+                extract_dir = os.path.join(OUTPUT_DIR, os.path.splitext(filename)[0])
+                os.makedirs(extract_dir, exist_ok=True)
+                
+                try:
+                    with zipfile.ZipFile(local_path, 'r') as zip_ref:
+                        zip_ref.extractall(extract_dir)
+                    logging.info(f"[✓] Extracted to: {extract_dir}")
+                    
+                    # Upload all extracted contents
+                    upload_directory_contents(extract_dir)
+                    
+                    # Cleanup
+                    shutil.rmtree(extract_dir)
+                    os.remove(local_path)
+                    logging.info(f"[✓] Cleaned up extracted files and zip")
+                except zipfile.BadZipFile:
+                    logging.error(f"[!] Invalid zip file: {filename}")
+                    os.remove(local_path)
+            else:
+                # If not a zip, upload directly as before
+                upload_to_dataset(local_path)
+                os.remove(local_path)
+
+        except Exception as e:
+            logging.error(f"[!] Error with {direct_download_link}: {e}")
+
+    logging.info("✅ All files processed.")
+
+@app.get("/")
+def stay_alive():
+    return {"msg": "Running"}
+
+@app.get("/health")
+def healthcheck():
+    return {"healthy": True}
+
+# === FastAPI Lifespan ===
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    logging.info("🚀 Starting FastAPI download-uploader microservice...")
+    task = asyncio.create_task(downloader_worker())
+    yield
+    task.cancel()
+    logging.info("🛑 Shutting down microservice.")
+
+# === FastAPI App ===
+app = FastAPI(lifespan=lifespan)
+# Re-assign app with lifespan logic
+app.router.lifespan_context = lifespan

requirements.txt

@@ -0,0 +1,5 @@
+requests
+huggingface_hub
+bs4
+fastapi
+uvicorn