""" Download the Mendeley Indian Spices dataset (DOI: 10.17632/vg77y9rtjb.3, CC BY 4.0). 19 zip files, ~1.22 GB total. Parallel download with retry + verification. After download, automatically extracts each zip into Indian_Spices//. """ import concurrent.futures as cf import hashlib import sys import time import zipfile from pathlib import Path import urllib.request import urllib.error # (filename, expected_bytes, download_url) FILES = [ ("Asafoetida.zip", 31509663, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/50f86deb-a723-47c0-9953-1fe489c61006/file_downloaded"), ("Bay Leaf.zip", 93346153, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/0f7dba50-d7ed-447a-9bb8-b04cef972bfd/file_downloaded"), ("Black Cardamom.zip", 21213820, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/a8261233-66ca-418d-a4ab-1433121ca3fa/file_downloaded"), ("Black Pepper.zip", 34335704, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/eb37d31e-b7d2-4b09-84d3-f6dfcfeec21a/file_downloaded"), ("Caraway seeds.zip", 213101833, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/43cb8369-571b-4c13-b6bc-debd13a189b9/file_downloaded"), ("Cinnamom stick.zip", 31757294, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/8fc24765-da8a-4b5f-8637-92a56044c83e/file_downloaded"), ("Cloves.zip", 124727438, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/edbd041b-e837-4432-929c-ea4ea253e472/file_downloaded"), ("Coriander Seeds.zip", 53003314, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/a170c01d-abc3-49e2-a927-2904f2840a45/file_downloaded"), ("Cubeb Pepper.zip", 27061024, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/548175f1-2c3b-4186-b524-c0a2c91c6b34/file_downloaded"), ("Cumin seeds.zip", 93137559, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/64a41c45-1d1d-4434-a149-23b6165bb585/file_downloaded"), ("Dry Ginger.zip", 83594994, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/3443a826-0a85-4e3e-bf80-d9574e1ffcbd/file_downloaded"), ("Dry red Chilly.zip", 66070750, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/6bf6f5ea-657f-4909-868b-3e9b76aca6ee/file_downloaded"), ("Fennel seeds.zip", 69818271, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/60f8f871-61ac-400b-970e-c2e1c771b03b/file_downloaded"), ("Green Cardamom.zip", 52079597, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/fba3646a-4d7e-4e1b-b04b-9a1feb6daca7/file_downloaded"), ("Mace.zip", 48481004, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/0fd32fc8-2704-4dbe-a508-e61667e27df7/file_downloaded"), ("Nutmeg.zip", 20227587, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/a77f3500-064c-4b10-9e04-20f543439334/file_downloaded"), ("Poppy Seeds.zip", 39959406, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/937c3767-2e91-4da5-a038-fe054d837a0f/file_downloaded"), ("Star Anise.zip", 36471796, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/e793420b-1f73-4ce9-8eef-b29267542ec9/file_downloaded"), ("Stone Flowers.zip", 77814235, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/285d0718-49ba-4635-a2f4-bf37e09bd822/file_downloaded"), ] ROOT = Path(__file__).parent / "Indian_Spices" ZIP_DIR = ROOT / "_zips" ROOT.mkdir(exist_ok=True, parents=True) ZIP_DIR.mkdir(exist_ok=True, parents=True) def _human(n: int) -> str: for unit in ("B", "KB", "MB", "GB"): if n < 1024: return f"{n:.1f}{unit}" n /= 1024 return f"{n:.1f}TB" def _download_one(item, max_retries: int = 3) -> tuple[str, bool, str]: filename, expected_size, url = item out_path = ZIP_DIR / filename if out_path.exists() and out_path.stat().st_size == expected_size: return filename, True, "cached" for attempt in range(1, max_retries + 1): try: t0 = time.time() req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) with urllib.request.urlopen(req, timeout=60) as resp, open(out_path, "wb") as fh: while chunk := resp.read(1 << 20): # 1 MB chunks fh.write(chunk) actual = out_path.stat().st_size if actual != expected_size: if attempt < max_retries: out_path.unlink(missing_ok=True) continue return filename, False, f"size mismatch {actual} vs {expected_size}" dt = time.time() - t0 return filename, True, f"{_human(actual)} in {dt:.1f}s" except (urllib.error.URLError, TimeoutError, ConnectionError) as e: if attempt < max_retries: time.sleep(2 ** attempt) continue return filename, False, f"error: {e}" return filename, False, "exhausted retries" def _extract_one(filename: str) -> tuple[str, int]: zip_path = ZIP_DIR / filename class_name = zip_path.stem # "Asafoetida" from "Asafoetida.zip" out_dir = ROOT / class_name if out_dir.exists() and any(out_dir.iterdir()): # Count images count = sum(1 for p in out_dir.rglob("*") if p.suffix.lower() in {".jpg", ".jpeg", ".png"}) return class_name, count out_dir.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(zip_path, "r") as zf: zf.extractall(out_dir) count = sum(1 for p in out_dir.rglob("*") if p.suffix.lower() in {".jpg", ".jpeg", ".png"}) return class_name, count def main(): total_expected = sum(s for _, s, _ in FILES) print(f"Mendeley Indian Spices — {len(FILES)} files, total {_human(total_expected)}") print(f"Output: {ROOT}\n") # ── Parallel download ──────────────────────────────────────────────── print("Downloading...") fails = [] with cf.ThreadPoolExecutor(max_workers=4) as ex: futs = {ex.submit(_download_one, item): item for item in FILES} for fut in cf.as_completed(futs): name, ok, msg = fut.result() mark = "OK " if ok else "FAIL" print(f" [{mark}] {name:22s} {msg}") if not ok: fails.append((name, msg)) if fails: print(f"\n{len(fails)} downloads failed:") for name, msg in fails: print(f" - {name}: {msg}") sys.exit(1) # ── Extract ────────────────────────────────────────────────────────── print("\nExtracting...") counts = {} with cf.ThreadPoolExecutor(max_workers=4) as ex: futs = {ex.submit(_extract_one, fname): fname for fname, _, _ in FILES} for fut in cf.as_completed(futs): cls, n = fut.result() counts[cls] = n print(f" {cls:22s} {n:5d} images") print(f"\nTotal extracted: {sum(counts.values())} images across {len(counts)} classes.") print(f"Expected by paper: 10,991 images across 19 classes.") if __name__ == "__main__": main()