| """
|
| Download the Mendeley Indian Spices dataset (DOI: 10.17632/vg77y9rtjb.3, CC BY 4.0).
|
|
|
| 19 zip files, ~1.22 GB total. Parallel download with retry + verification.
|
| After download, automatically extracts each zip into Indian_Spices/<class_name>/.
|
| """
|
| import concurrent.futures as cf
|
| import hashlib
|
| import sys
|
| import time
|
| import zipfile
|
| from pathlib import Path
|
|
|
| import urllib.request
|
| import urllib.error
|
|
|
|
|
| FILES = [
|
| ("Asafoetida.zip", 31509663, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/50f86deb-a723-47c0-9953-1fe489c61006/file_downloaded"),
|
| ("Bay Leaf.zip", 93346153, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/0f7dba50-d7ed-447a-9bb8-b04cef972bfd/file_downloaded"),
|
| ("Black Cardamom.zip", 21213820, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/a8261233-66ca-418d-a4ab-1433121ca3fa/file_downloaded"),
|
| ("Black Pepper.zip", 34335704, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/eb37d31e-b7d2-4b09-84d3-f6dfcfeec21a/file_downloaded"),
|
| ("Caraway seeds.zip", 213101833, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/43cb8369-571b-4c13-b6bc-debd13a189b9/file_downloaded"),
|
| ("Cinnamom stick.zip", 31757294, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/8fc24765-da8a-4b5f-8637-92a56044c83e/file_downloaded"),
|
| ("Cloves.zip", 124727438, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/edbd041b-e837-4432-929c-ea4ea253e472/file_downloaded"),
|
| ("Coriander Seeds.zip", 53003314, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/a170c01d-abc3-49e2-a927-2904f2840a45/file_downloaded"),
|
| ("Cubeb Pepper.zip", 27061024, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/548175f1-2c3b-4186-b524-c0a2c91c6b34/file_downloaded"),
|
| ("Cumin seeds.zip", 93137559, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/64a41c45-1d1d-4434-a149-23b6165bb585/file_downloaded"),
|
| ("Dry Ginger.zip", 83594994, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/3443a826-0a85-4e3e-bf80-d9574e1ffcbd/file_downloaded"),
|
| ("Dry red Chilly.zip", 66070750, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/6bf6f5ea-657f-4909-868b-3e9b76aca6ee/file_downloaded"),
|
| ("Fennel seeds.zip", 69818271, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/60f8f871-61ac-400b-970e-c2e1c771b03b/file_downloaded"),
|
| ("Green Cardamom.zip", 52079597, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/fba3646a-4d7e-4e1b-b04b-9a1feb6daca7/file_downloaded"),
|
| ("Mace.zip", 48481004, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/0fd32fc8-2704-4dbe-a508-e61667e27df7/file_downloaded"),
|
| ("Nutmeg.zip", 20227587, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/a77f3500-064c-4b10-9e04-20f543439334/file_downloaded"),
|
| ("Poppy Seeds.zip", 39959406, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/937c3767-2e91-4da5-a038-fe054d837a0f/file_downloaded"),
|
| ("Star Anise.zip", 36471796, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/e793420b-1f73-4ce9-8eef-b29267542ec9/file_downloaded"),
|
| ("Stone Flowers.zip", 77814235, "https://data.mendeley.com/public-files/datasets/vg77y9rtjb/files/285d0718-49ba-4635-a2f4-bf37e09bd822/file_downloaded"),
|
| ]
|
|
|
| ROOT = Path(__file__).parent / "Indian_Spices"
|
| ZIP_DIR = ROOT / "_zips"
|
| ROOT.mkdir(exist_ok=True, parents=True)
|
| ZIP_DIR.mkdir(exist_ok=True, parents=True)
|
|
|
|
|
| def _human(n: int) -> str:
|
| for unit in ("B", "KB", "MB", "GB"):
|
| if n < 1024:
|
| return f"{n:.1f}{unit}"
|
| n /= 1024
|
| return f"{n:.1f}TB"
|
|
|
|
|
| def _download_one(item, max_retries: int = 3) -> tuple[str, bool, str]:
|
| filename, expected_size, url = item
|
| out_path = ZIP_DIR / filename
|
|
|
| if out_path.exists() and out_path.stat().st_size == expected_size:
|
| return filename, True, "cached"
|
|
|
| for attempt in range(1, max_retries + 1):
|
| try:
|
| t0 = time.time()
|
| req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
| with urllib.request.urlopen(req, timeout=60) as resp, open(out_path, "wb") as fh:
|
| while chunk := resp.read(1 << 20):
|
| fh.write(chunk)
|
| actual = out_path.stat().st_size
|
| if actual != expected_size:
|
| if attempt < max_retries:
|
| out_path.unlink(missing_ok=True)
|
| continue
|
| return filename, False, f"size mismatch {actual} vs {expected_size}"
|
| dt = time.time() - t0
|
| return filename, True, f"{_human(actual)} in {dt:.1f}s"
|
| except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
|
| if attempt < max_retries:
|
| time.sleep(2 ** attempt)
|
| continue
|
| return filename, False, f"error: {e}"
|
| return filename, False, "exhausted retries"
|
|
|
|
|
| def _extract_one(filename: str) -> tuple[str, int]:
|
| zip_path = ZIP_DIR / filename
|
| class_name = zip_path.stem
|
| out_dir = ROOT / class_name
|
| if out_dir.exists() and any(out_dir.iterdir()):
|
|
|
| count = sum(1 for p in out_dir.rglob("*") if p.suffix.lower() in {".jpg", ".jpeg", ".png"})
|
| return class_name, count
|
| out_dir.mkdir(parents=True, exist_ok=True)
|
| with zipfile.ZipFile(zip_path, "r") as zf:
|
| zf.extractall(out_dir)
|
| count = sum(1 for p in out_dir.rglob("*") if p.suffix.lower() in {".jpg", ".jpeg", ".png"})
|
| return class_name, count
|
|
|
|
|
| def main():
|
| total_expected = sum(s for _, s, _ in FILES)
|
| print(f"Mendeley Indian Spices — {len(FILES)} files, total {_human(total_expected)}")
|
| print(f"Output: {ROOT}\n")
|
|
|
|
|
| print("Downloading...")
|
| fails = []
|
| with cf.ThreadPoolExecutor(max_workers=4) as ex:
|
| futs = {ex.submit(_download_one, item): item for item in FILES}
|
| for fut in cf.as_completed(futs):
|
| name, ok, msg = fut.result()
|
| mark = "OK " if ok else "FAIL"
|
| print(f" [{mark}] {name:22s} {msg}")
|
| if not ok:
|
| fails.append((name, msg))
|
|
|
| if fails:
|
| print(f"\n{len(fails)} downloads failed:")
|
| for name, msg in fails:
|
| print(f" - {name}: {msg}")
|
| sys.exit(1)
|
|
|
|
|
| print("\nExtracting...")
|
| counts = {}
|
| with cf.ThreadPoolExecutor(max_workers=4) as ex:
|
| futs = {ex.submit(_extract_one, fname): fname for fname, _, _ in FILES}
|
| for fut in cf.as_completed(futs):
|
| cls, n = fut.result()
|
| counts[cls] = n
|
| print(f" {cls:22s} {n:5d} images")
|
|
|
| print(f"\nTotal extracted: {sum(counts.values())} images across {len(counts)} classes.")
|
| print(f"Expected by paper: 10,991 images across 19 classes.")
|
|
|
|
|
| if __name__ == "__main__":
|
| main()
|
|
|
