Spaces:
Running
Running
Download e.py from AkshithSurya/spherex-projects: direct link, hf CLI and curl.
- Browser
- Download file 4.54 kB
-
https://huggingface.co/spaces/AkshithSurya/spherex-projects/resolve/main/e.py
- Command line
-
hf download hf://spaces/AkshithSurya/spherex-projects/e.py
-
curl -L -o e.py https://huggingface.co/spaces/AkshithSurya/spherex-projects/resolve/main/e.py
4.54 kB
| import os | |
| import sys | |
| import time | |
| import requests | |
| from astropy.coordinates import SkyCoord | |
| import astropy.units as u | |
| from astroquery.ipac.irsa import Irsa | |
| from concurrent.futures import ThreadPoolExecutor, as_completed | |
| from threading import Lock | |
| RA_DEG = 127.69444 | |
| DEC_DEG = -39.17760 | |
| RADIUS_DEG = 0.05 | |
| COLLECTION = "spherex_qr2" | |
| BAND_FILTER = None | |
| MAX_IMAGES = None | |
| MAX_WORKERS = 16 | |
| MAX_RETRIES = 3 | |
| CHUNK_SIZE = 1024 * 1024 | |
| TIMEOUT = 180 | |
| DATA_DIR = "spherex_data" | |
| FORCE_REDOWNLOAD = False | |
| progress_lock = Lock() | |
| progress = {"done": 0, "failed": 0, "skipped": 0, "total": 0} | |
| def log_progress(tag): | |
| with progress_lock: | |
| progress[tag] += 1 | |
| completed = progress["done"] + progress["failed"] + progress["skipped"] | |
| print( | |
| f"[{completed}/{progress['total']}] " | |
| f"done={progress['done']} skipped={progress['skipped']} failed={progress['failed']}", | |
| end="\r", | |
| ) | |
| def get_records(): | |
| pos = SkyCoord(ra=RA_DEG * u.deg, dec=DEC_DEG * u.deg) | |
| print(f"Querying IRSA SIA: RA={RA_DEG}, Dec={DEC_DEG}, radius={RADIUS_DEG} deg ...") | |
| result = Irsa.query_sia(pos=(pos, RADIUS_DEG * u.deg), collection=COLLECTION) | |
| print(f"Found {len(result)} raw results") | |
| urls = [ | |
| uri if str(uri).startswith("http") else f"https://irsa.ipac.caltech.edu/{uri}" | |
| for uri in result["access_url"] | |
| ] | |
| bands = ( | |
| [str(b) for b in result["energy_bandpassname"]] | |
| if "energy_bandpassname" in result.colnames | |
| else [""] * len(urls) | |
| ) | |
| records = list(zip(urls, bands)) | |
| if BAND_FILTER: | |
| records = [(u_, b) for u_, b in records if BAND_FILTER in b] | |
| print(f"Filtered to band '{BAND_FILTER}': {len(records)} images") | |
| if MAX_IMAGES: | |
| records = records[:MAX_IMAGES] | |
| print(f"Limited to first {MAX_IMAGES} images for this run") | |
| return records | |
| def band_folder(band): | |
| safe = band.replace("/", "_").strip() if band else "unknown_band" | |
| folder = os.path.join(DATA_DIR, safe or "unknown_band") | |
| os.makedirs(folder, exist_ok=True) | |
| return folder | |
| def download_one(url, band): | |
| filename = os.path.basename(url) | |
| dest_folder = band_folder(band) | |
| dest_path = os.path.join(dest_folder, filename) | |
| if not FORCE_REDOWNLOAD and os.path.exists(dest_path): | |
| try: | |
| head = requests.head(url, timeout=30) | |
| remote_size = int(head.headers.get("Content-Length", 0)) | |
| local_size = os.path.getsize(dest_path) | |
| if remote_size and local_size == remote_size: | |
| log_progress("skipped") | |
| return dest_path | |
| except Exception: | |
| log_progress("skipped") | |
| return dest_path | |
| tmp_path = dest_path + ".part" | |
| last_err = None | |
| for attempt in range(1, MAX_RETRIES + 1): | |
| try: | |
| with requests.get(url, stream=True, timeout=TIMEOUT) as r: | |
| r.raise_for_status() | |
| with open(tmp_path, "wb") as f: | |
| for chunk in r.iter_content(chunk_size=CHUNK_SIZE): | |
| if chunk: | |
| f.write(chunk) | |
| os.replace(tmp_path, dest_path) | |
| log_progress("done") | |
| return dest_path | |
| except Exception as e: | |
| last_err = e | |
| if os.path.exists(tmp_path): | |
| os.remove(tmp_path) | |
| time.sleep(2 * attempt) | |
| log_progress("failed") | |
| print(f"\nFAILED after {MAX_RETRIES} attempts: {url}\n reason: {last_err}") | |
| return None | |
| def main(): | |
| os.makedirs(DATA_DIR, exist_ok=True) | |
| if FORCE_REDOWNLOAD: | |
| print("FORCE_REDOWNLOAD is True — existing files will be overwritten from scratch.\n") | |
| records = get_records() | |
| if not records: | |
| print("No images found for this query. Try widening RADIUS_DEG.") | |
| sys.exit(1) | |
| progress["total"] = len(records) | |
| print(f"\nDownloading {len(records)} .fits files with {MAX_WORKERS} parallel workers...\n") | |
| downloaded_paths = [] | |
| with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: | |
| futures = {executor.submit(download_one, url, band): (url, band) for url, band in records} | |
| for future in as_completed(futures): | |
| path = future.result() | |
| if path: | |
| downloaded_paths.append(path) | |
| print(f"\n\nDownload complete: {len(downloaded_paths)}/{len(records)} .fits files saved.") | |
| print(f"All files kept as raw .fits under: {os.path.abspath(DATA_DIR)}") | |
| if __name__ == "__main__": | |
| main() | |