File size: 3,093 Bytes
ceae2aa c676c52 ceae2aa a2f9200 ceae2aa a2f9200 ceae2aa 775c52f ceae2aa c676c52 9f8c7ac 7c9b282 a2f9200 ceae2aa 7fa0c2c ceae2aa 805923a a2f9200 ceae2aa a2f9200 ceae2aa a2f9200 ceae2aa a2f9200 9e95f6c a2f9200 c676c52 a2f9200 c676c52 a2f9200 c676c52 a2f9200 c676c52 a2f9200 ceae2aa a2f9200 7fa0c2c 775c52f a2f9200 ceae2aa a2f9200 805923a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 |
import os
import time
import requests
from urllib.parse import urlparse
from huggingface_hub import upload_file
from fastapi import FastAPI
from contextlib import asynccontextmanager
import asyncio
import logging
# === CONFIGURATION ===
HF_TOKEN = os.environ.get("HF_TOKEN")
REPO_ID = "Fred808/BG1"
DATA_PATH = "Blenders"
OUTPUT_DIR = "batch_downloads"
DOWNLOAD_URLS = [
"https://ww6.zeroupload.xyz/d6d34ba916295d15826814b6500d25e9/BlenderLaunchPad_DownloadPirate.com.part4.rar?download_token=71eb49f728ee2096211102b585bd61b734dc9e407c3978a899a44084ee277a46"
]
DELAY_BETWEEN_DOWNLOADS = 12 # seconds
# === Setup Logging ===
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
# === Prepare output folder ===
os.makedirs(OUTPUT_DIR, exist_ok=True)
app = FastAPI()
# === DUMMY ROUTE TO KEEP SERVER HEALTHY ===
@app.get("/")
def keep_alive():
return {"status": "running"}
# === Upload Function ===
def upload_to_dataset(filepath):
try:
upload_file(
path_or_fileobj=filepath,
path_in_repo=f"{DATA_PATH}/{os.path.basename(filepath)}",
repo_id=REPO_ID,
repo_type="dataset",
token=HF_TOKEN
)
logging.info(f"[↑] Uploaded: {filepath}")
except Exception as e:
logging.error(f"[!] Upload failed: {filepath} — {e}")
# === Background Worker ===
async def downloader_worker():
for direct_download_link in DOWNLOAD_URLS:
logging.info("[*] Waiting before next download...")
await asyncio.sleep(DELAY_BETWEEN_DOWNLOADS)
try:
logging.info(f"[*] Downloading from: {direct_download_link}")
filename = os.path.basename(urlparse(direct_download_link).path)
if not filename or "." not in filename:
filename = "downloaded_file_" + str(int(time.time()))
local_path = os.path.join(OUTPUT_DIR, filename)
logging.info(f"[*] Saving to: {local_path}")
with requests.get(direct_download_link, stream=True) as r:
r.raise_for_status()
with open(local_path, "wb") as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
logging.info(f"[✓] Downloaded: {filename}")
upload_to_dataset(local_path)
os.remove(local_path)
except Exception as e:
logging.error(f"[!] Error with {direct_download_link}: {e}")
logging.info("✅ All files processed.")
@app.get("/")
def stay_alive():
return {"msg": "Running"}
@app.get("/health")
def healthcheck():
return {"healthy": True}
# === FastAPI Lifespan ===
@asynccontextmanager
async def lifespan(app: FastAPI):
logging.info("🚀 Starting FastAPI download-uploader microservice...")
task = asyncio.create_task(downloader_worker())
yield
task.cancel()
logging.info("🛑 Shutting down microservice.")
# === FastAPI App ===
app = FastAPI(lifespan=lifespan)
# Re-assign app with lifespan logic
app.router.lifespan_context = lifespan |