File size: 1,796 Bytes
6fa9282 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | """Download a versioned scPerturb count matrix and verify its source checksum."""
from pathlib import Path
import argparse, hashlib, os
import requests
FILES = {
"norman": ("NormanWeissman2019_filtered.h5ad", "c870e6967d91c017d9da827bab183cd6"),
"replogle_k562": (
"ReplogleWeissman2022_K562_essential.h5ad",
"d8cba17576d1a8afc0f7d71b79cad0f7",
),
}
def md5(path):
h = hashlib.md5()
with open(path, "rb") as f:
for block in iter(lambda: f.read(8 * 1024**2), b""):
h.update(block)
return h.hexdigest()
def download(dataset, output):
name, digest = FILES[dataset]
out = Path(output)
out.mkdir(parents=True, exist_ok=True)
dest = out / name
if dest.exists():
if md5(dest) != digest:
raise ValueError(f"Checksum mismatch: {dest}")
return dest
part = dest.with_suffix(".h5ad.partial")
offset = part.stat().st_size if part.exists() else 0
url = f"https://zenodo.org/records/10044268/files/{name}?download=1"
with requests.get(
url,
stream=True,
headers={"Range": f"bytes={offset}-"} if offset else {},
timeout=(30, 120),
) as response:
response.raise_for_status()
append = offset > 0 and response.status_code == 206
with open(part, "ab" if append else "wb") as f:
for chunk in response.iter_content(8 * 1024**2):
f.write(chunk)
if md5(part) != digest:
raise ValueError(f"Source checksum mismatch: {part}")
os.replace(part, dest)
return dest
if __name__ == "__main__":
p = argparse.ArgumentParser()
p.add_argument("dataset", choices=FILES)
p.add_argument("--output", default="data/raw")
a = p.parse_args()
print(download(a.dataset, a.output))
|