Buckets:
| import argparse | |
| import hashlib | |
| import json | |
| import re | |
| import zlib | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path, PurePosixPath | |
| import requests | |
| from bs4 import BeautifulSoup | |
| from remotezip import RemoteZip | |
| from prepare import split_groups, storage_guard | |
| url = "https://zenodo.org/api/records/7223446/files/OpenEarthMap.zip/content" | |
| def normalized(value: str) -> str: | |
| return re.sub(r"[^a-z]", "", value.lower()) | |
| def select_oem(names: list[str], licenses: dict, limit: int) -> list[dict]: | |
| available = set(names) | |
| regions = {} | |
| for name in sorted(names): | |
| if "/images/" not in name or not name.endswith(".tif"): | |
| continue | |
| region = PurePosixPath(name).parts[-3] | |
| mask = name.replace("/images/", "/labels/") | |
| if region not in licenses or mask not in available: | |
| continue | |
| regions.setdefault(region, []).append({"region": region, "image_member": name, "mask_member": mask}) | |
| for rows in regions.values(): | |
| rows.sort(key=lambda row: hashlib.sha256(row["image_member"].encode()).digest()) | |
| selected = [] | |
| ordered = sorted(regions, key=lambda region: hashlib.sha256(region.encode()).digest()) | |
| while len(selected) < limit and any(regions.values()): | |
| for region in ordered: | |
| if regions[region] and len(selected) < limit: | |
| selected.append(regions[region].pop()) | |
| return selected | |
| def acquire(limit: int) -> None: | |
| root = Path("datasets") | |
| metadata = root / "metadata" | |
| metadata.mkdir(parents=True, exist_ok=True) | |
| response = requests.get("https://open-earth-map.org/attribution.html", timeout=45) | |
| response.raise_for_status() | |
| (metadata / "openearthmap_attribution.html").write_text(response.text, encoding="utf-8") | |
| source_rows = {} | |
| for row in BeautifulSoup(response.text, "html.parser").select("tr"): | |
| cells = row.select("td") | |
| if len(cells) < 5: | |
| continue | |
| region, country, origin, provider, license_name = [cell.get_text(" ", strip=True) for cell in cells[:5]] | |
| if license_name not in {"CC BY 4.0", "CC BY-SA 4.0", "CC BY-NC-SA 4.0", "DL-DE-BY-2.0", "Public domain"}: | |
| continue | |
| if origin == "xBD": | |
| continue | |
| label_license = "CC BY-NC-SA 4.0" if license_name == "Public domain" else license_name | |
| item = { | |
| "region_name": region, "country": country, "upstream_dataset": origin, | |
| "provider": provider, "image_license": license_name, "license": label_license, | |
| "license_group": "academic_only" if "NC" in label_license else "commercial_compatible", | |
| "attribution_url": "https://open-earth-map.org/attribution.html", | |
| "license_urls": [link["href"] for link in cells[4].select("a[href]")], | |
| } | |
| source_rows.setdefault(normalized(region), []).append(item) | |
| with RemoteZip(url, timeout=90) as archive: | |
| names = archive.namelist() | |
| archive_rows = [{"name": info.filename, "bytes": info.file_size, "compressed_bytes": info.compress_size, "crc32": info.CRC} for info in archive.infolist()] | |
| (metadata / "openearthmap_archive_members.json").write_text(json.dumps(archive_rows), encoding="utf-8") | |
| available_regions = {PurePosixPath(name).parts[-3] for name in names if "/images/" in name and name.endswith(".tif")} | |
| licenses = {} | |
| for region in available_regions: | |
| matches = source_rows.get(normalized(region)) | |
| if matches and len({item["license"] for item in matches}) == 1: | |
| licenses[region] = dict(matches[0], providers=matches) | |
| selected = select_oem(names, licenses, limit) | |
| groups = {region: "oem:" + licenses[region]["country"] + ":" + normalized(licenses[region]["region_name"]) for region in {row["region"] for row in selected}} | |
| splits = split_groups(list(groups.values())) | |
| original_splits = {} | |
| for split in ("train", "val", "test"): | |
| member = f"OpenEarthMap_wo_xBD/{split}.txt" | |
| for name in archive.read(member).decode().splitlines(): | |
| original_splits[PurePosixPath(name.strip()).name] = split | |
| for row in selected: | |
| row.update(licenses[row["region"]], group=groups[row["region"]], split=splits[groups[row["region"]]], original_split=original_splits.get(PurePosixPath(row["image_member"]).name)) | |
| selected_bytes = sum(archive.getinfo(row[key]).file_size for row in selected for key in ("image_member", "mask_member")) | |
| workspace_budget = storage_guard(Path.cwd(), 20000000000) | |
| if workspace_budget.used + selected_bytes + 5500000000 > 19000000000: | |
| raise OSError("selection leaves insufficient room under 20 GB budget") | |
| (metadata / "oem_selection.json").write_text(json.dumps(selected, indent=2), encoding="utf-8") | |
| print(f"selected {len(selected)} pairs across {len(groups)} region folders; raw estimate {selected_bytes / 1e9:.2f} GB", flush=True) | |
| def worker(rows: list[dict]) -> list[dict]: | |
| completed = [] | |
| with RemoteZip(url, timeout=90) as archive: | |
| for index, row in enumerate(rows): | |
| item = dict(row, files=[]) | |
| for key, folder in (("image_member", "images"), ("mask_member", "labels")): | |
| member = row[key] | |
| info = archive.getinfo(member) | |
| destination = root / "raw" / "openearthmap" / row["region"] / folder / PurePosixPath(member).name | |
| destination.parent.mkdir(parents=True, exist_ok=True) | |
| payload = destination.read_bytes() if destination.exists() else None | |
| if payload is not None and (len(payload) != info.file_size or zlib.crc32(payload) != info.CRC): | |
| raise ValueError(f"cached file integrity mismatch: {destination}") | |
| if payload is None: | |
| payload = archive.read(member) | |
| if len(payload) != info.file_size or zlib.crc32(payload) != info.CRC: | |
| raise ValueError(f"ZIP member integrity mismatch: {member}") | |
| temporary = destination.with_suffix(".part") | |
| workspace_budget.reserve(len(payload)) | |
| temporary.write_bytes(payload) | |
| temporary.replace(destination) | |
| item[key.replace("_member", "_path")] = str(destination) | |
| item["files"].append({"archive_member": member, "path": str(destination), "bytes": len(payload), "crc32": f"{info.CRC:08x}", "sha256": hashlib.sha256(payload).hexdigest()}) | |
| completed.append(item) | |
| if (index + 1) % 10 == 0: | |
| print(f"worker progress: {index + 1}/{len(rows)} pairs", flush=True) | |
| return completed | |
| with ThreadPoolExecutor(max_workers=4) as pool: | |
| batches = list(pool.map(worker, [selected[i::4] for i in range(4)])) | |
| records = sorted([row for batch in batches for row in batch], key=lambda row: row["image_member"]) | |
| (metadata / "oem_downloaded.json").write_text(json.dumps(records, indent=2), encoding="utf-8") | |
| print(f"verified {len(records)} OpenEarthMap pairs; full-archive MD5 not claimed for range-selected data", flush=True) | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--oem-limit", type=int, default=400) | |
| args = parser.parse_args() | |
| if args.oem_limit < 3: | |
| parser.error("at least three samples required") | |
| acquire(args.oem_limit) | |
Xet Storage Details
- Size:
- 7.72 kB
- Xet hash:
- 8d835780538a122d9681a6e02764ae13876b5dce08b5056924ab6311c8f6c3d3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.