Spaces:
Paused
Paused
Download scripts/fetch_data.py from Kamlesh21/preprocessing-lab: direct link, hf CLI and curl.
- Browser
- Download file 5.63 kB
-
https://huggingface.co/spaces/Kamlesh21/preprocessing-lab/resolve/main/scripts/fetch_data.py
- Command line
-
hf download hf://spaces/Kamlesh21/preprocessing-lab/scripts/fetch_data.py
-
curl -L -o fetch_data.py https://huggingface.co/spaces/Kamlesh21/preprocessing-lab/resolve/main/scripts/fetch_data.py
5.63 kB
| """Download the demo images once, so the demo runs offline afterwards. | |
| - Classification: one test image per breed from Oxford-IIIT Pet (via the Hugging Face datasets server). | |
| - Segmentation: multi-object images from Ultralytics' coco128-seg (a 128-image COCO subset with instance | |
| polygons, which give the ground-truth counts). | |
| Every image here was checked: on the clean pipeline, ResNet50 names each pet correctly (87-100%) and | |
| YOLO11s-seg counts every class in each COCO photo exactly. That way "break it" starts from a right answer. | |
| Writes backend/data/manifest.json. Safe to re-run: existing files are skipped. | |
| """ | |
| from __future__ import annotations | |
| import io | |
| import json | |
| import urllib.parse | |
| import urllib.request | |
| import zipfile | |
| from collections import Counter | |
| from pathlib import Path | |
| DATA = Path(__file__).resolve().parents[1] / "data" | |
| HF_ROWS = "https://datasets-server.huggingface.co/rows" | |
| COCO128 = "https://github.com/ultralytics/assets/releases/download/v0.0.0/coco128-seg.zip" | |
| PET_BREEDS = { # label index in timm/oxford-iiit-pet -> (file name, the matching ImageNet class) | |
| 25: ("pug", "pug"), 29: ("samoyed", "Samoyed"), 4: ("beagle", "beagle"), | |
| 28: ("saint_bernard", "Saint Bernard"), 8: ("boxer", "boxer"), 10: ("chihuahua", "Chihuahua"), | |
| 23: ("persian", "Persian cat"), 11: ("egyptian_mau", "Egyptian cat"), | |
| } | |
| # coco128-seg images on which YOLO11s-seg counts every class exactly (checked through the Segment line: RGB, letterbox 640, conf 0.25). | |
| COCO_PICKS = ["000000000049", "000000000074", "000000000143", "000000000597", "000000000136", "000000000599"] | |
| COCO_NAMES = [ | |
| "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", | |
| "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", | |
| "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", | |
| "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", | |
| "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", | |
| "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", | |
| "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", | |
| "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", | |
| "scissors", "teddy bear", "hair drier", "toothbrush", | |
| ] | |
| def get(url: str) -> bytes: | |
| req = urllib.request.Request(url, headers={"User-Agent": "image-preprocessing-demo"}) | |
| with urllib.request.urlopen(req, timeout=120) as r: | |
| return r.read() | |
| def pet_image_urls(wanted: set[int]) -> dict[int, str]: | |
| """Scan the test split page by page until one image per wanted breed is found.""" | |
| found: dict[int, str] = {} | |
| for offset in range(0, 3700, 100): | |
| q = urllib.parse.urlencode({"dataset": "timm/oxford-iiit-pet", "config": "default", | |
| "split": "test", "offset": offset, "length": 100}) | |
| for r in json.loads(get(f"{HF_ROWS}?{q}"))["rows"]: | |
| label = r["row"]["label"] | |
| if label in wanted and label not in found: | |
| found[label] = r["row"]["image"]["src"] | |
| if len(found) == len(wanted): | |
| break | |
| return found | |
| def fetch_pets() -> list[dict]: | |
| out = [] | |
| (DATA / "pets").mkdir(parents=True, exist_ok=True) | |
| missing = {label for label, (breed, _) in PET_BREEDS.items() if not (DATA / "pets" / f"{breed}.jpg").exists()} | |
| urls = pet_image_urls(missing) if missing else {} | |
| for label, (breed, imagenet) in PET_BREEDS.items(): | |
| path = DATA / "pets" / f"{breed}.jpg" | |
| if label in urls: | |
| path.write_bytes(get(urls[label])) | |
| print("pets:", breed) | |
| if not path.exists(): | |
| continue | |
| out.append({ | |
| "id": f"pet-{breed}", "use_case": "classify", "file": f"pets/{breed}.jpg", | |
| "title": breed.replace("_", " ").title(), "label": imagenet, | |
| "license": "CC BY-SA 4.0", | |
| "source": "Oxford-IIIT Pet Dataset (Parkhi et al., 2012), test split, via huggingface.co/datasets/timm/oxford-iiit-pet", | |
| }) | |
| return out | |
| def fetch_coco() -> list[dict]: | |
| (DATA / "coco").mkdir(parents=True, exist_ok=True) | |
| zf = zipfile.ZipFile(io.BytesIO(get(COCO128))) | |
| out = [] | |
| for stem in COCO_PICKS: | |
| labels = next(f for f in zf.namelist() if f.endswith(f"/labels/train2017/{stem}.txt")) | |
| counts = Counter(COCO_NAMES[int(line.split()[0])] for line in zf.read(labels).decode().splitlines() if line.strip()) | |
| image = next(f for f in zf.namelist() if f.endswith(f"/images/train2017/{stem}.jpg")) | |
| (DATA / "coco" / f"{stem}.jpg").write_bytes(zf.read(image)) | |
| out.append({ | |
| "id": f"coco-{stem}", "use_case": "segment", "file": f"coco/{stem}.jpg", | |
| "title": ", ".join(f"{v} {k}" for k, v in counts.most_common()), | |
| "gt_counts": dict(counts), "license": "COCO annotations CC BY 4.0; images under their Flickr licenses", | |
| "source": "COCO 2017 via Ultralytics coco128-seg (train2017 subset, so the model has seen these images)", | |
| }) | |
| print("coco:", stem, dict(counts)) | |
| return out | |
| def main() -> None: | |
| DATA.mkdir(parents=True, exist_ok=True) | |
| images = fetch_pets() + fetch_coco() | |
| (DATA / "manifest.json").write_text(json.dumps({"images": images}, indent=2)) | |
| print(f"wrote manifest with {len(images)} images") | |
| if __name__ == "__main__": | |
| main() | |