Bl4ckSpaces commited on
Commit
8a75b5b
·
verified ·
1 Parent(s): 0c2b9e7

Release v16.0.2: Windows Support, Retry Logic, and CLI

Browse files
Files changed (2) hide show
  1. setup.py +8 -3
  2. speedster/__init__.py +109 -72
setup.py CHANGED
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
2
 
3
  setup(
4
  name="speedster",
5
- version="16.0.1", # UPDATED VERSION
6
  packages=find_packages(),
7
  install_requires=[
8
  "aiohttp>=3.9.0",
@@ -10,8 +10,13 @@ setup(
10
  "nest_asyncio",
11
  "requests"
12
  ],
 
 
 
 
 
13
  author="Bl4ckSpaces",
14
- description="Ultra-fast, asynchronous, memory-efficient universal downloader (Nitro Edition).",
15
- keywords="downloader, aiohttp, async, civitai, huggingface, speedster",
16
  python_requires=">=3.8",
17
  )
 
2
 
3
  setup(
4
  name="speedster",
5
+ version="16.0.2", # Major Update: CLI & Windows Support
6
  packages=find_packages(),
7
  install_requires=[
8
  "aiohttp>=3.9.0",
 
10
  "nest_asyncio",
11
  "requests"
12
  ],
13
+ entry_points={
14
+ 'console_scripts': [
15
+ 'speedster=speedster:cli_main', # Mendaftarkan command 'speedster'
16
+ ],
17
+ },
18
  author="Bl4ckSpaces",
19
+ description="Universal cross-platform downloader (Windows/Linux/Colab) with connection retry logic.",
20
+ keywords="downloader, aiohttp, async, civitai, huggingface, speedster, cli",
21
  python_requires=">=3.8",
22
  )
speedster/__init__.py CHANGED
@@ -1,7 +1,9 @@
1
  import os
 
2
  import asyncio
3
  import aiohttp
4
  import urllib.parse
 
5
  from tqdm.asyncio import tqdm
6
  import nest_asyncio
7
 
@@ -9,99 +11,134 @@ import nest_asyncio
9
  nest_asyncio.apply()
10
 
11
  class Speedster:
12
- def __init__(self, num_threads=16, chunk_size_mb=2):
13
  self.num_threads = num_threads
14
- self.chunk_size = chunk_size_mb * 1024 * 1024
15
-
 
 
16
  self.default_headers = {
17
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
18
  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
19
  "Accept-Language": "en-US,en;q=0.5",
20
  }
21
 
 
 
 
 
 
 
 
 
 
 
22
  async def _fetch_chunk(self, session, url, start, end, fd, headers, pbar):
23
  chunk_headers = headers.copy()
24
  chunk_headers['Range'] = f'bytes={start}-{end}'
25
-
26
- try:
27
- async with session.get(url, headers=chunk_headers, timeout=60) as response:
28
- response.raise_for_status()
29
- offset = start
30
- async for chunk in response.content.iter_chunked(self.chunk_size):
31
- if chunk:
32
- os.pwrite(fd, chunk, offset)
33
- offset += len(chunk)
34
- pbar.update(len(chunk))
35
- except Exception as e:
36
- # print(f"Retry chunk {start}...")
37
- pass
 
 
 
 
 
 
 
 
38
 
39
  async def _download_logic(self, url, dest_path, headers):
40
  async with aiohttp.ClientSession() as session:
41
  # 1. Get Metadata
42
- async with session.get(url, headers=headers, allow_redirects=True) as response:
43
- final_url = response.url
44
- total_size = int(response.headers.get('Content-Length', 0))
45
-
46
- # Smart Filename Detection
47
- filename = "downloaded_file.bin"
48
- if os.path.isdir(dest_path):
49
- if 'Content-Disposition' in response.headers:
50
- try:
51
- cd = response.headers['Content-Disposition']
52
- filename = cd.split('filename=')[1].strip('"')
53
- except:
54
- filename = os.path.basename(urllib.parse.urlparse(str(final_url)).path)
55
- else:
56
- path_filename = os.path.basename(urllib.parse.urlparse(str(final_url)).path)
57
- if path_filename: filename = path_filename
58
-
59
- dest_path = os.path.join(dest_path, filename)
60
-
61
- if total_size == 0:
62
- print(f"❌ Server rejected size request.")
63
- return
64
-
65
- print(f"🚀 [Speedster Nitro] Target: {dest_path} | Size: {total_size / (1024**3):.2f} GB")
66
-
67
- # 2. Prepare Disk
68
- fd = os.open(dest_path, os.O_RDWR | os.O_CREAT)
69
- os.ftruncate(fd, total_size)
70
-
71
- # 3. Create Tasks
72
- chunk_size = total_size // self.num_threads
73
- chunks = []
74
- for i in range(self.num_threads):
75
- start = i * chunk_size
76
- end = total_size - 1 if i == self.num_threads - 1 else (start + chunk_size - 1)
77
- chunks.append((start, end))
78
-
79
- pbar = tqdm(total=total_size, unit='iB', unit_scale=True, unit_divisor=1024, desc="⚡ SPEEDSTER")
80
- tasks = [asyncio.create_task(self._fetch_chunk(session, final_url, s, e, fd, headers, pbar)) for s, e in chunks]
81
-
82
- # 4. AWAIT ALL TASKS (Blocking)
83
- await asyncio.gather(*tasks)
84
-
85
- pbar.close()
86
- os.close(fd)
87
- print("✅ Download Selesai!")
88
-
89
- def download(self, url, dest_path="/content", token=None):
 
 
 
 
 
 
 
90
  headers = self.default_headers.copy()
91
  if token:
92
  headers["Authorization"] = f"Bearer {token}"
93
-
94
- # --- FIX UTAMA DI SINI ---
95
  loop = asyncio.get_event_loop()
96
  if loop.is_running():
97
- # Jika ada loop berjalan (Colab), kita patch dan paksa tunggu
98
  nest_asyncio.apply()
99
  loop.run_until_complete(self._download_logic(url, dest_path, headers))
100
  else:
101
- # Jika script python biasa
102
  asyncio.run(self._download_logic(url, dest_path, headers))
103
 
104
- # Global Wrapper
105
- def download(url, dest_path="/content", token=None, threads=16):
106
- engine = Speedster(num_threads=threads)
107
- engine.download(url, dest_path, token)
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
+ import sys
3
  import asyncio
4
  import aiohttp
5
  import urllib.parse
6
+ import argparse
7
  from tqdm.asyncio import tqdm
8
  import nest_asyncio
9
 
 
11
  nest_asyncio.apply()
12
 
13
  class Speedster:
14
+ def __init__(self, num_threads=16, chunk_size_mb=2, max_retries=5):
15
  self.num_threads = num_threads
16
+ self.chunk_size = chunk_size_mb * 1024 * 1024
17
+ self.max_retries = max_retries
18
+ self.is_windows = os.name == 'nt' # Deteksi Windows
19
+
20
  self.default_headers = {
21
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
22
  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
23
  "Accept-Language": "en-US,en;q=0.5",
24
  }
25
 
26
+ # --- CROSS-PLATFORM WRITER ---
27
+ def _write_data(self, fd, data, offset):
28
+ if self.is_windows:
29
+ # Windows Fallback: Lseek + Write (Blocking but safe)
30
+ os.lseek(fd, offset, 0)
31
+ os.write(fd, data)
32
+ else:
33
+ # Linux/Mac/Android: Pwrite (Atomic & Fast)
34
+ os.pwrite(fd, data, offset)
35
+
36
  async def _fetch_chunk(self, session, url, start, end, fd, headers, pbar):
37
  chunk_headers = headers.copy()
38
  chunk_headers['Range'] = f'bytes={start}-{end}'
39
+
40
+ # --- RETRY LOGIC (NYAWA CADANGAN) ---
41
+ for attempt in range(self.max_retries):
42
+ try:
43
+ async with session.get(url, headers=chunk_headers, timeout=60) as response:
44
+ response.raise_for_status()
45
+ offset = start
46
+ async for chunk in response.content.iter_chunked(self.chunk_size):
47
+ if chunk:
48
+ self._write_data(fd, chunk, offset)
49
+ offset += len(chunk)
50
+ pbar.update(len(chunk))
51
+ return # Sukses, keluar dari fungsi
52
+ except Exception as e:
53
+ if attempt < self.max_retries - 1:
54
+ # Tunggu sebentar sebelum coba lagi (Exponential Backoff)
55
+ await asyncio.sleep(1 * (attempt + 1))
56
+ else:
57
+ # Nyawa habis
58
+ tqdm.write(f"❌ Gagal pada chunk {start}-{end} setelah {self.max_retries} kali coba. Error: {e}")
59
+ raise e
60
 
61
  async def _download_logic(self, url, dest_path, headers):
62
  async with aiohttp.ClientSession() as session:
63
  # 1. Get Metadata
64
+ try:
65
+ async with session.get(url, headers=headers, allow_redirects=True) as response:
66
+ final_url = response.url
67
+ total_size = int(response.headers.get('Content-Length', 0))
68
+
69
+ # Smart Filename
70
+ filename = "downloaded_file.bin"
71
+ if os.path.isdir(dest_path):
72
+ if 'Content-Disposition' in response.headers:
73
+ try:
74
+ cd = response.headers['Content-Disposition']
75
+ filename = cd.split('filename=')[1].strip('"')
76
+ except:
77
+ filename = os.path.basename(urllib.parse.urlparse(str(final_url)).path)
78
+ else:
79
+ path_filename = os.path.basename(urllib.parse.urlparse(str(final_url)).path)
80
+ if path_filename: filename = path_filename
81
+
82
+ # Clean filename from weird URL params
83
+ filename = filename.split('?')[0]
84
+ dest_path = os.path.join(dest_path, filename)
85
+
86
+ if total_size == 0:
87
+ print(f" Server tidak memberikan ukuran file. Tidak bisa multi-thread.")
88
+ return
89
+
90
+ print(f"🚀 [Speedster Universal] Target: {dest_path}")
91
+ print(f"📦 Size: {total_size / (1024**3):.2f} GB | Threads: {self.num_threads} | OS: {'Windows' if self.is_windows else 'Linux/Unix'}")
92
+
93
+ # 2. Prepare Disk
94
+ # Mode wb+ untuk binary write update
95
+ fd = os.open(dest_path, os.O_RDWR | os.O_CREAT | os.O_BINARY if self.is_windows else os.O_RDWR | os.O_CREAT)
96
+ os.ftruncate(fd, total_size)
97
+
98
+ # 3. Create Tasks
99
+ chunk_size = total_size // self.num_threads
100
+ chunks = []
101
+ for i in range(self.num_threads):
102
+ start = i * chunk_size
103
+ end = total_size - 1 if i == self.num_threads - 1 else (start + chunk_size - 1)
104
+ chunks.append((start, end))
105
+
106
+ pbar = tqdm(total=total_size, unit='iB', unit_scale=True, unit_divisor=1024, desc="⚡ DOWNLOADING")
107
+ tasks = [asyncio.create_task(self._fetch_chunk(session, final_url, s, e, fd, headers, pbar)) for s, e in chunks]
108
+
109
+ await asyncio.gather(*tasks)
110
+
111
+ pbar.close()
112
+ os.close(fd)
113
+ print("✅ Download Selesai!")
114
+
115
+ except Exception as e:
116
+ print(f"❌ Critical Error: {e}")
117
+
118
+ def download(self, url, dest_path=".", token=None):
119
  headers = self.default_headers.copy()
120
  if token:
121
  headers["Authorization"] = f"Bearer {token}"
122
+
 
123
  loop = asyncio.get_event_loop()
124
  if loop.is_running():
 
125
  nest_asyncio.apply()
126
  loop.run_until_complete(self._download_logic(url, dest_path, headers))
127
  else:
 
128
  asyncio.run(self._download_logic(url, dest_path, headers))
129
 
130
+ # --- CLI ENTRY POINT ---
131
+ def cli_main():
132
+ parser = argparse.ArgumentParser(description="Speedster v16.0.2 - Universal Downloader")
133
+ parser.add_argument("url", help="URL file/model yang ingin didownload")
134
+ parser.add_argument("--out", "-o", default=".", help="Folder tujuan (Default: folder saat ini)")
135
+ parser.add_argument("--token", "-t", help="Bearer token (Khusus Civitai/HuggingFace Private)")
136
+ parser.add_argument("--threads", type=int, default=16, help="Jumlah koneksi simultan (Default: 16)")
137
+
138
+ args = parser.parse_args()
139
+
140
+ engine = Speedster(num_threads=args.threads)
141
+ engine.download(args.url, args.out, args.token)
142
+
143
+ if __name__ == "__main__":
144
+ cli_main()