Hugging Face's logo

Spaces:
factorstudios
/
Zeup
Running

App Files Community
Zeup
File size: 5,159 Bytes
058701f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8368afd
fde72b4
058701f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
import os
import time
import requests
import zipfile
import shutil
from urllib.parse import urlparse
from huggingface_hub import upload_file
from fastapi import FastAPI
from contextlib import asynccontextmanager
import asyncio
import logging

# === CONFIGURATION ===
HF_TOKEN = os.environ.get("HF_TOKEN")
REPO_ID = "factorstudios/Pipeline"
DATA_PATH = "Blenders"
OUTPUT_DIR = "batch_downloads"
DOWNLOAD_URLS = [
   "https://ww2.zeroupload.xyz/bb1ddbafa3e7b9d024332f7a1ac7aa44/Class101Juan_LearniPadDrawing_DownloadPirate.com.part2.rar?download_token=b20d7f68c1b7fea792efc126f55b824e26a51032259c01bbd9b72d5d308060f0"
    ]
DELAY_BETWEEN_DOWNLOADS = 12  # seconds

# === Setup Logging ===
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")

# === Prepare output folder ===
os.makedirs(OUTPUT_DIR, exist_ok=True)

app = FastAPI()

# === DUMMY ROUTE TO KEEP SERVER HEALTHY ===
@app.get("/")
def keep_alive():
    return {"status": "running"}

# === Upload Function ===
def upload_to_dataset(filepath):
    try:
        upload_file(
            path_or_fileobj=filepath,
            path_in_repo=f"{DATA_PATH}/{os.path.basename(filepath)}",
            repo_id=REPO_ID,
            repo_type="dataset",
            token=HF_TOKEN
        )
        logging.info(f"[↑] Uploaded: {filepath}")
    except Exception as e:
        logging.error(f"[!] Upload failed: {filepath} — {e}")

# === Upload Directory Contents ===
def upload_directory_contents(directory):
    try:
        for root, dirs, files in os.walk(directory):
            for file in files:
                filepath = os.path.join(root, file)
                relative_path = os.path.relpath(filepath, directory)
                upload_file(
                    path_or_fileobj=filepath,
                    path_in_repo=f"{DATA_PATH}/{relative_path}",
                    repo_id=REPO_ID,
                    repo_type="dataset",
                    token=HF_TOKEN
                )
                logging.info(f"[↑] Uploaded: {relative_path}")
    except Exception as e:
        logging.error(f"[!] Upload directory failed: {directory} — {e}")

# === Background Worker ===
async def downloader_worker():
    for direct_download_link in DOWNLOAD_URLS:
        logging.info("[*] Waiting before next download...")
        await asyncio.sleep(DELAY_BETWEEN_DOWNLOADS)

        try:
            logging.info(f"[*] Downloading from: {direct_download_link}")
            filename = os.path.basename(urlparse(direct_download_link).path)
            if not filename or "." not in filename:
                filename = "downloaded_file_" + str(int(time.time()))

            local_path = os.path.join(OUTPUT_DIR, filename)
            logging.info(f"[*] Saving to: {local_path}")

            with requests.get(direct_download_link, stream=True) as r:
                r.raise_for_status()
                with open(local_path, "wb") as f:
                    for chunk in r.iter_content(chunk_size=8192):
                        f.write(chunk)

            logging.info(f"[✓] Downloaded: {filename}")
            
            # Check if file is a zip file
            if filename.lower().endswith('.zip'):
                logging.info(f"[*] Extracting zip file: {filename}")
                extract_dir = os.path.join(OUTPUT_DIR, os.path.splitext(filename)[0])
                os.makedirs(extract_dir, exist_ok=True)
                
                try:
                    with zipfile.ZipFile(local_path, 'r') as zip_ref:
                        zip_ref.extractall(extract_dir)
                    logging.info(f"[✓] Extracted to: {extract_dir}")
                    
                    # Upload all extracted contents
                    upload_directory_contents(extract_dir)
                    
                    # Cleanup
                    shutil.rmtree(extract_dir)
                    os.remove(local_path)
                    logging.info(f"[✓] Cleaned up extracted files and zip")
                except zipfile.BadZipFile:
                    logging.error(f"[!] Invalid zip file: {filename}")
                    os.remove(local_path)
            else:
                # If not a zip, upload directly as before
                upload_to_dataset(local_path)
                os.remove(local_path)

        except Exception as e:
            logging.error(f"[!] Error with {direct_download_link}: {e}")

    logging.info("✅ All files processed.")

@app.get("/")
def stay_alive():
    return {"msg": "Running"}

@app.get("/health")
def healthcheck():
    return {"healthy": True}

# === FastAPI Lifespan ===
@asynccontextmanager
async def lifespan(app: FastAPI):
    logging.info("🚀 Starting FastAPI download-uploader microservice...")
    task = asyncio.create_task(downloader_worker())
    yield
    task.cancel()
    logging.info("🛑 Shutting down microservice.")

# === FastAPI App ===
app = FastAPI(lifespan=lifespan)
# Re-assign app with lifespan logic
app.router.lifespan_context = lifespan