Release Speedster v16 Nitro (Auto-Upload)

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+sample_data/mnist_test.csv filter=lfs diff=lfs merge=lfs -text
+sample_data/mnist_train_small.csv filter=lfs diff=lfs merge=lfs -text

sample_data/README.md

@@ -0,0 +1,19 @@
+This directory includes a few sample datasets to get you started.
+
+*   `california_housing_data*.csv` is California housing data from the 1990 US
+    Census; more information is available at:
+    https://docs.google.com/document/d/e/2PACX-1vRhYtsvc5eOR2FWNCwaBiKL6suIOrxJig8LcSBbmCbyYsayia_DvPOOBlXZ4CAlQ5nlDD8kTaIDRwrN/pub
+
+*   `mnist_*.csv` is a small sample of the
+    [MNIST database](https://en.wikipedia.org/wiki/MNIST_database), which is
+    described at: http://yann.lecun.com/exdb/mnist/
+
+*   `anscombe.json` contains a copy of
+    [Anscombe's quartet](https://en.wikipedia.org/wiki/Anscombe%27s_quartet); it
+    was originally described in
+
+    Anscombe, F. J. (1973). 'Graphs in Statistical Analysis'. American
+    Statistician. 27 (1): 17-21. JSTOR 2682899.
+
+    and our copy was prepared by the
+    [vega_datasets library](https://github.com/altair-viz/vega_datasets/blob/4f67bdaad10f45e3549984e17e1b3088c731503d/vega_datasets/_data/anscombe.json).

sample_data/anscombe.json

@@ -0,0 +1,49 @@
+[
+  {"Series":"I", "X":10.0, "Y":8.04},
+  {"Series":"I", "X":8.0, "Y":6.95},
+  {"Series":"I", "X":13.0, "Y":7.58},
+  {"Series":"I", "X":9.0, "Y":8.81},
+  {"Series":"I", "X":11.0, "Y":8.33},
+  {"Series":"I", "X":14.0, "Y":9.96},
+  {"Series":"I", "X":6.0, "Y":7.24},
+  {"Series":"I", "X":4.0, "Y":4.26},
+  {"Series":"I", "X":12.0, "Y":10.84},
+  {"Series":"I", "X":7.0, "Y":4.81},
+  {"Series":"I", "X":5.0, "Y":5.68},
+
+  {"Series":"II", "X":10.0, "Y":9.14},
+  {"Series":"II", "X":8.0, "Y":8.14},
+  {"Series":"II", "X":13.0, "Y":8.74},
+  {"Series":"II", "X":9.0, "Y":8.77},
+  {"Series":"II", "X":11.0, "Y":9.26},
+  {"Series":"II", "X":14.0, "Y":8.10},
+  {"Series":"II", "X":6.0, "Y":6.13},
+  {"Series":"II", "X":4.0, "Y":3.10},
+  {"Series":"II", "X":12.0, "Y":9.13},
+  {"Series":"II", "X":7.0, "Y":7.26},
+  {"Series":"II", "X":5.0, "Y":4.74},
+
+  {"Series":"III", "X":10.0, "Y":7.46},
+  {"Series":"III", "X":8.0, "Y":6.77},
+  {"Series":"III", "X":13.0, "Y":12.74},
+  {"Series":"III", "X":9.0, "Y":7.11},
+  {"Series":"III", "X":11.0, "Y":7.81},
+  {"Series":"III", "X":14.0, "Y":8.84},
+  {"Series":"III", "X":6.0, "Y":6.08},
+  {"Series":"III", "X":4.0, "Y":5.39},
+  {"Series":"III", "X":12.0, "Y":8.15},
+  {"Series":"III", "X":7.0, "Y":6.42},
+  {"Series":"III", "X":5.0, "Y":5.73},
+
+  {"Series":"IV", "X":8.0, "Y":6.58},
+  {"Series":"IV", "X":8.0, "Y":5.76},
+  {"Series":"IV", "X":8.0, "Y":7.71},
+  {"Series":"IV", "X":8.0, "Y":8.84},
+  {"Series":"IV", "X":8.0, "Y":8.47},
+  {"Series":"IV", "X":8.0, "Y":7.04},
+  {"Series":"IV", "X":8.0, "Y":5.25},
+  {"Series":"IV", "X":19.0, "Y":12.50},
+  {"Series":"IV", "X":8.0, "Y":5.56},
+  {"Series":"IV", "X":8.0, "Y":7.91},
+  {"Series":"IV", "X":8.0, "Y":6.89}
+]

sample_data/mnist_test.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51c292478d94ec3a01461bdfa82eb0885d262eb09e615679b2d69dedb6ad09e7
+size 18289443

sample_data/mnist_train_small.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef64781aa03180f4f5ce504314f058f5d0227277df86060473d973cf43b033e
+size 36523880

setup.py

@@ -0,0 +1,17 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="speedster",
+    version="16.0.0",
+    packages=find_packages(),
+    install_requires=[
+        "aiohttp>=3.9.0",
+        "tqdm>=4.66.0",
+        "nest_asyncio",
+        "requests"
+    ],
+    author="Bl4ckSpaces",
+    description="Ultra-fast, asynchronous, memory-efficient universal downloader (Nitro Edition).",
+    keywords="downloader, aiohttp, async, civitai, huggingface, speedster",
+    python_requires=">=3.8",
+)

speedster/__init__.py

@@ -0,0 +1,108 @@
+import os
+import asyncio
+import aiohttp
+import urllib.parse
+from tqdm.asyncio import tqdm
+import nest_asyncio
+
+# Apply nest_asyncio to allow running asyncio in Jupyter/Colab notebooks
+nest_asyncio.apply()
+
+class Speedster:
+    def __init__(self, num_threads=16, chunk_size_mb=2):
+        self.num_threads = num_threads
+        self.chunk_size = chunk_size_mb * 1024 * 1024 # Buffer in bytes
+        
+        # Headers samaran agar dianggap browser (Anti-Block)
+        self.default_headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+        }
+
+    async def _fetch_chunk(self, session, url, start, end, fd, headers, pbar):
+        chunk_headers = headers.copy()
+        chunk_headers['Range'] = f'bytes={start}-{end}'
+        
+        try:
+            async with session.get(url, headers=chunk_headers, timeout=60) as response:
+                response.raise_for_status()
+                offset = start
+                
+                async for chunk in response.content.iter_chunked(self.chunk_size):
+                    if chunk:
+                        # [CORE MAGIC] os.pwrite langsung ke disk tanpa locking python
+                        os.pwrite(fd, chunk, offset)
+                        offset += len(chunk)
+                        pbar.update(len(chunk))
+        except Exception as e:
+            # Silent retry mechanism could be added here
+            print(f"⚠️ Chunk Error: {e}")
+            raise
+
+    async def _download_logic(self, url, dest_path, headers):
+        async with aiohttp.ClientSession() as session:
+            # 1. Get Metadata
+            async with session.get(url, headers=headers, allow_redirects=True) as response:
+                final_url = response.url
+                total_size = int(response.headers.get('Content-Length', 0))
+                
+                if os.path.isdir(dest_path):
+                    if 'Content-Disposition' in response.headers:
+                        try:
+                            cd = response.headers['Content-Disposition']
+                            filename = cd.split('filename=')[1].strip('"')
+                        except:
+                            filename = os.path.basename(urllib.parse.urlparse(str(final_url)).path)
+                    else:
+                        filename = os.path.basename(urllib.parse.urlparse(str(final_url)).path)
+                    
+                    if not filename: filename = "downloaded_file.bin"
+                    dest_path = os.path.join(dest_path, filename)
+
+            if total_size == 0:
+                print(f"❌ Server menolak memberikan ukuran file. Download gagal.")
+                return
+
+            print(f"🚀 [Speedster Nitro] Target: {dest_path} | Size: {total_size / (1024**3):.2f} GB")
+
+            # 2. Chunking
+            chunk_size = total_size // self.num_threads
+            chunks = []
+            for i in range(self.num_threads):
+                start = i * chunk_size
+                end = total_size - 1 if i == self.num_threads - 1 else (start + chunk_size - 1)
+                chunks.append((start, end))
+
+            # 3. Allocating Disk (Zero-Fill)
+            fd = os.open(dest_path, os.O_RDWR | os.O_CREAT)
+            os.ftruncate(fd, total_size)
+
+            # 4. Fire Async Workers
+            pbar = tqdm(total=total_size, unit='iB', unit_scale=True, unit_divisor=1024, desc="⚡ SPEEDSTER")
+            tasks = []
+            for start, end in chunks:
+                task = asyncio.create_task(self._fetch_chunk(session, final_url, start, end, fd, headers, pbar))
+                tasks.append(task)
+            
+            await asyncio.gather(*tasks)
+            pbar.close()
+            os.close(fd)
+            print("✅ Download Selesai!")
+
+    def download(self, url, dest_path="/content", token=None):
+        headers = self.default_headers.copy()
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+        
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            # Jika sudah dalam event loop (seperti di Colab)
+            asyncio.create_task(self._download_logic(url, dest_path, headers))
+        else:
+            asyncio.run(self._download_logic(url, dest_path, headers))
+
+# Wrapper Global
+def download(url, dest_path="/content", token=None, threads=16):
+    engine = Speedster(num_threads=threads)
+    engine.download(url, dest_path, token)