Release v16.0.2: Windows Support, Retry Logic, and CLI
Browse files- setup.py +8 -3
- speedster/__init__.py +109 -72
setup.py
CHANGED
|
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
|
|
| 2 |
|
| 3 |
setup(
|
| 4 |
name="speedster",
|
| 5 |
-
version="16.0.
|
| 6 |
packages=find_packages(),
|
| 7 |
install_requires=[
|
| 8 |
"aiohttp>=3.9.0",
|
|
@@ -10,8 +10,13 @@ setup(
|
|
| 10 |
"nest_asyncio",
|
| 11 |
"requests"
|
| 12 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
author="Bl4ckSpaces",
|
| 14 |
-
description="
|
| 15 |
-
keywords="downloader, aiohttp, async, civitai, huggingface, speedster",
|
| 16 |
python_requires=">=3.8",
|
| 17 |
)
|
|
|
|
| 2 |
|
| 3 |
setup(
|
| 4 |
name="speedster",
|
| 5 |
+
version="16.0.2", # Major Update: CLI & Windows Support
|
| 6 |
packages=find_packages(),
|
| 7 |
install_requires=[
|
| 8 |
"aiohttp>=3.9.0",
|
|
|
|
| 10 |
"nest_asyncio",
|
| 11 |
"requests"
|
| 12 |
],
|
| 13 |
+
entry_points={
|
| 14 |
+
'console_scripts': [
|
| 15 |
+
'speedster=speedster:cli_main', # Mendaftarkan command 'speedster'
|
| 16 |
+
],
|
| 17 |
+
},
|
| 18 |
author="Bl4ckSpaces",
|
| 19 |
+
description="Universal cross-platform downloader (Windows/Linux/Colab) with connection retry logic.",
|
| 20 |
+
keywords="downloader, aiohttp, async, civitai, huggingface, speedster, cli",
|
| 21 |
python_requires=">=3.8",
|
| 22 |
)
|
speedster/__init__.py
CHANGED
|
@@ -1,7 +1,9 @@
|
|
| 1 |
import os
|
|
|
|
| 2 |
import asyncio
|
| 3 |
import aiohttp
|
| 4 |
import urllib.parse
|
|
|
|
| 5 |
from tqdm.asyncio import tqdm
|
| 6 |
import nest_asyncio
|
| 7 |
|
|
@@ -9,99 +11,134 @@ import nest_asyncio
|
|
| 9 |
nest_asyncio.apply()
|
| 10 |
|
| 11 |
class Speedster:
|
| 12 |
-
def __init__(self, num_threads=16, chunk_size_mb=2):
|
| 13 |
self.num_threads = num_threads
|
| 14 |
-
self.chunk_size = chunk_size_mb * 1024 * 1024
|
| 15 |
-
|
|
|
|
|
|
|
| 16 |
self.default_headers = {
|
| 17 |
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
| 18 |
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
| 19 |
"Accept-Language": "en-US,en;q=0.5",
|
| 20 |
}
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
async def _fetch_chunk(self, session, url, start, end, fd, headers, pbar):
|
| 23 |
chunk_headers = headers.copy()
|
| 24 |
chunk_headers['Range'] = f'bytes={start}-{end}'
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
async def _download_logic(self, url, dest_path, headers):
|
| 40 |
async with aiohttp.ClientSession() as session:
|
| 41 |
# 1. Get Metadata
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
if
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
headers = self.default_headers.copy()
|
| 91 |
if token:
|
| 92 |
headers["Authorization"] = f"Bearer {token}"
|
| 93 |
-
|
| 94 |
-
# --- FIX UTAMA DI SINI ---
|
| 95 |
loop = asyncio.get_event_loop()
|
| 96 |
if loop.is_running():
|
| 97 |
-
# Jika ada loop berjalan (Colab), kita patch dan paksa tunggu
|
| 98 |
nest_asyncio.apply()
|
| 99 |
loop.run_until_complete(self._download_logic(url, dest_path, headers))
|
| 100 |
else:
|
| 101 |
-
# Jika script python biasa
|
| 102 |
asyncio.run(self._download_logic(url, dest_path, headers))
|
| 103 |
|
| 104 |
-
#
|
| 105 |
-
def
|
| 106 |
-
|
| 107 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import os
|
| 2 |
+
import sys
|
| 3 |
import asyncio
|
| 4 |
import aiohttp
|
| 5 |
import urllib.parse
|
| 6 |
+
import argparse
|
| 7 |
from tqdm.asyncio import tqdm
|
| 8 |
import nest_asyncio
|
| 9 |
|
|
|
|
| 11 |
nest_asyncio.apply()
|
| 12 |
|
| 13 |
class Speedster:
|
| 14 |
+
def __init__(self, num_threads=16, chunk_size_mb=2, max_retries=5):
|
| 15 |
self.num_threads = num_threads
|
| 16 |
+
self.chunk_size = chunk_size_mb * 1024 * 1024
|
| 17 |
+
self.max_retries = max_retries
|
| 18 |
+
self.is_windows = os.name == 'nt' # Deteksi Windows
|
| 19 |
+
|
| 20 |
self.default_headers = {
|
| 21 |
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
| 22 |
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
| 23 |
"Accept-Language": "en-US,en;q=0.5",
|
| 24 |
}
|
| 25 |
|
| 26 |
+
# --- CROSS-PLATFORM WRITER ---
|
| 27 |
+
def _write_data(self, fd, data, offset):
|
| 28 |
+
if self.is_windows:
|
| 29 |
+
# Windows Fallback: Lseek + Write (Blocking but safe)
|
| 30 |
+
os.lseek(fd, offset, 0)
|
| 31 |
+
os.write(fd, data)
|
| 32 |
+
else:
|
| 33 |
+
# Linux/Mac/Android: Pwrite (Atomic & Fast)
|
| 34 |
+
os.pwrite(fd, data, offset)
|
| 35 |
+
|
| 36 |
async def _fetch_chunk(self, session, url, start, end, fd, headers, pbar):
|
| 37 |
chunk_headers = headers.copy()
|
| 38 |
chunk_headers['Range'] = f'bytes={start}-{end}'
|
| 39 |
+
|
| 40 |
+
# --- RETRY LOGIC (NYAWA CADANGAN) ---
|
| 41 |
+
for attempt in range(self.max_retries):
|
| 42 |
+
try:
|
| 43 |
+
async with session.get(url, headers=chunk_headers, timeout=60) as response:
|
| 44 |
+
response.raise_for_status()
|
| 45 |
+
offset = start
|
| 46 |
+
async for chunk in response.content.iter_chunked(self.chunk_size):
|
| 47 |
+
if chunk:
|
| 48 |
+
self._write_data(fd, chunk, offset)
|
| 49 |
+
offset += len(chunk)
|
| 50 |
+
pbar.update(len(chunk))
|
| 51 |
+
return # Sukses, keluar dari fungsi
|
| 52 |
+
except Exception as e:
|
| 53 |
+
if attempt < self.max_retries - 1:
|
| 54 |
+
# Tunggu sebentar sebelum coba lagi (Exponential Backoff)
|
| 55 |
+
await asyncio.sleep(1 * (attempt + 1))
|
| 56 |
+
else:
|
| 57 |
+
# Nyawa habis
|
| 58 |
+
tqdm.write(f"❌ Gagal pada chunk {start}-{end} setelah {self.max_retries} kali coba. Error: {e}")
|
| 59 |
+
raise e
|
| 60 |
|
| 61 |
async def _download_logic(self, url, dest_path, headers):
|
| 62 |
async with aiohttp.ClientSession() as session:
|
| 63 |
# 1. Get Metadata
|
| 64 |
+
try:
|
| 65 |
+
async with session.get(url, headers=headers, allow_redirects=True) as response:
|
| 66 |
+
final_url = response.url
|
| 67 |
+
total_size = int(response.headers.get('Content-Length', 0))
|
| 68 |
+
|
| 69 |
+
# Smart Filename
|
| 70 |
+
filename = "downloaded_file.bin"
|
| 71 |
+
if os.path.isdir(dest_path):
|
| 72 |
+
if 'Content-Disposition' in response.headers:
|
| 73 |
+
try:
|
| 74 |
+
cd = response.headers['Content-Disposition']
|
| 75 |
+
filename = cd.split('filename=')[1].strip('"')
|
| 76 |
+
except:
|
| 77 |
+
filename = os.path.basename(urllib.parse.urlparse(str(final_url)).path)
|
| 78 |
+
else:
|
| 79 |
+
path_filename = os.path.basename(urllib.parse.urlparse(str(final_url)).path)
|
| 80 |
+
if path_filename: filename = path_filename
|
| 81 |
+
|
| 82 |
+
# Clean filename from weird URL params
|
| 83 |
+
filename = filename.split('?')[0]
|
| 84 |
+
dest_path = os.path.join(dest_path, filename)
|
| 85 |
+
|
| 86 |
+
if total_size == 0:
|
| 87 |
+
print(f"❌ Server tidak memberikan ukuran file. Tidak bisa multi-thread.")
|
| 88 |
+
return
|
| 89 |
+
|
| 90 |
+
print(f"🚀 [Speedster Universal] Target: {dest_path}")
|
| 91 |
+
print(f"📦 Size: {total_size / (1024**3):.2f} GB | Threads: {self.num_threads} | OS: {'Windows' if self.is_windows else 'Linux/Unix'}")
|
| 92 |
+
|
| 93 |
+
# 2. Prepare Disk
|
| 94 |
+
# Mode wb+ untuk binary write update
|
| 95 |
+
fd = os.open(dest_path, os.O_RDWR | os.O_CREAT | os.O_BINARY if self.is_windows else os.O_RDWR | os.O_CREAT)
|
| 96 |
+
os.ftruncate(fd, total_size)
|
| 97 |
+
|
| 98 |
+
# 3. Create Tasks
|
| 99 |
+
chunk_size = total_size // self.num_threads
|
| 100 |
+
chunks = []
|
| 101 |
+
for i in range(self.num_threads):
|
| 102 |
+
start = i * chunk_size
|
| 103 |
+
end = total_size - 1 if i == self.num_threads - 1 else (start + chunk_size - 1)
|
| 104 |
+
chunks.append((start, end))
|
| 105 |
+
|
| 106 |
+
pbar = tqdm(total=total_size, unit='iB', unit_scale=True, unit_divisor=1024, desc="⚡ DOWNLOADING")
|
| 107 |
+
tasks = [asyncio.create_task(self._fetch_chunk(session, final_url, s, e, fd, headers, pbar)) for s, e in chunks]
|
| 108 |
+
|
| 109 |
+
await asyncio.gather(*tasks)
|
| 110 |
+
|
| 111 |
+
pbar.close()
|
| 112 |
+
os.close(fd)
|
| 113 |
+
print("✅ Download Selesai!")
|
| 114 |
+
|
| 115 |
+
except Exception as e:
|
| 116 |
+
print(f"❌ Critical Error: {e}")
|
| 117 |
+
|
| 118 |
+
def download(self, url, dest_path=".", token=None):
|
| 119 |
headers = self.default_headers.copy()
|
| 120 |
if token:
|
| 121 |
headers["Authorization"] = f"Bearer {token}"
|
| 122 |
+
|
|
|
|
| 123 |
loop = asyncio.get_event_loop()
|
| 124 |
if loop.is_running():
|
|
|
|
| 125 |
nest_asyncio.apply()
|
| 126 |
loop.run_until_complete(self._download_logic(url, dest_path, headers))
|
| 127 |
else:
|
|
|
|
| 128 |
asyncio.run(self._download_logic(url, dest_path, headers))
|
| 129 |
|
| 130 |
+
# --- CLI ENTRY POINT ---
|
| 131 |
+
def cli_main():
|
| 132 |
+
parser = argparse.ArgumentParser(description="Speedster v16.0.2 - Universal Downloader")
|
| 133 |
+
parser.add_argument("url", help="URL file/model yang ingin didownload")
|
| 134 |
+
parser.add_argument("--out", "-o", default=".", help="Folder tujuan (Default: folder saat ini)")
|
| 135 |
+
parser.add_argument("--token", "-t", help="Bearer token (Khusus Civitai/HuggingFace Private)")
|
| 136 |
+
parser.add_argument("--threads", type=int, default=16, help="Jumlah koneksi simultan (Default: 16)")
|
| 137 |
+
|
| 138 |
+
args = parser.parse_args()
|
| 139 |
+
|
| 140 |
+
engine = Speedster(num_threads=args.threads)
|
| 141 |
+
engine.download(args.url, args.out, args.token)
|
| 142 |
+
|
| 143 |
+
if __name__ == "__main__":
|
| 144 |
+
cli_main()
|