Spaces:
Running on Zero
Running on Zero
File size: 6,139 Bytes
a74054f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 | """
Download full ERA5 dataset (1960-2026) for all 27 stations.
Downloads year by year with separate requests for instantaneous vs accumulated variables.
NOTE: ERA5 separates instantaneous (temperature, wind) from accumulated
(precipitation, evaporation, radiation) variables. They must be requested separately.
"""
import cdsapi
import pandas as pd
import xarray as xr
import zipfile
import shutil
from pathlib import Path
# Load station coordinates
stations = pd.read_csv('datasets/station_list.csv')
lat_min = stations['lat'].min() - 0.1
lat_max = stations['lat'].max() + 0.1
lon_min = stations['lon'].min() - 0.1
lon_max = stations['lon'].max() + 0.1
print("="*80)
print("ERA5 FULL DOWNLOAD (1960-2026)")
print("="*80)
print(f"Study area: [{lat_min:.2f}, {lon_min:.2f}] to [{lat_max:.2f}, {lon_max:.2f}]")
print(f"Years: 1960-2026 (67 years)")
print(f"Output: datasets/safran/")
print("="*80)
# Split variables by type (instantaneous vs accumulated)
instantaneous_vars = [
'2m_temperature', # T_Q
'10m_u_component_of_wind', # FF_Q
'10m_v_component_of_wind', # FF_Q
]
accumulated_vars = [
'total_precipitation', # PRELIQ_Q
'potential_evaporation', # ETP_Q
'surface_solar_radiation_downwards', # DLI_Q / SSI_Q
'snowfall', # PRENEI_Q
'runoff', # RUNC_Q
]
def extract_netcdf(zip_file):
"""Extract NetCDF from ZIP archive."""
if zipfile.is_zipfile(zip_file):
with zipfile.ZipFile(zip_file, 'r') as zip_ref:
# Find all .nc files in the zip
nc_files = [f for f in zip_ref.namelist() if f.endswith('.nc')]
extracted_files = []
for nc_file in nc_files:
temp_file = zip_file.parent / f'temp_{nc_file}'
with zip_ref.open(nc_file) as source:
with open(temp_file, 'wb') as target:
shutil.copyfileobj(source, target)
extracted_files.append(temp_file)
zip_file.unlink()
return extracted_files
return [zip_file]
def merge_datasets(instant_file, accum_file, output_file):
"""Merge instantaneous and accumulated variable datasets."""
ds_instant = xr.open_dataset(instant_file)
ds_accum = xr.open_dataset(accum_file)
# Merge datasets
ds_merged = xr.merge([ds_instant, ds_accum])
ds_merged.to_netcdf(output_file)
ds_instant.close()
ds_accum.close()
instant_file.unlink()
accum_file.unlink()
c = cdsapi.Client()
output_dir = Path('datasets/safran')
output_dir.mkdir(exist_ok=True)
# Download year by year (1960-2026)
years = list(range(1960, 2027)) # 1960 to 2026 inclusive
failed_years = []
for year in years:
output_file = output_dir / f'era5_{year}.nc'
# Skip if already exists and contains all 8 variables
if output_file.exists():
try:
ds = xr.open_dataset(output_file)
if len(ds.data_vars) >= 8:
print(f"✓ {year} - Already downloaded with all variables, skipping")
ds.close()
continue
else:
print(f"⚠ {year} - Incomplete ({len(ds.data_vars)} vars), re-downloading")
ds.close()
output_file.unlink()
except:
output_file.unlink()
print(f"\nDownloading {year}...")
instant_file = output_dir / f'era5_{year}_instant.nc'
accum_file = output_dir / f'era5_{year}_accum.nc'
try:
# Download instantaneous variables
print(f" → Instantaneous variables...")
c.retrieve(
'reanalysis-era5-single-levels',
{
'product_type': 'reanalysis',
'format': 'netcdf',
'variable': instantaneous_vars,
'year': str(year),
'month': [f'{m:02d}' for m in range(1, 13)],
'day': [f'{d:02d}' for d in range(1, 32)],
'time': '12:00',
'area': [lat_max, lon_min, lat_min, lon_max],
},
str(instant_file)
)
# Extract if ZIP
extracted = extract_netcdf(instant_file)
if len(extracted) > 0:
instant_file = extracted[0]
# Download accumulated variables
print(f" → Accumulated variables...")
c.retrieve(
'reanalysis-era5-single-levels',
{
'product_type': 'reanalysis',
'format': 'netcdf',
'variable': accumulated_vars,
'year': str(year),
'month': [f'{m:02d}' for m in range(1, 13)],
'day': [f'{d:02d}' for d in range(1, 32)],
'time': '12:00',
'area': [lat_max, lon_min, lat_min, lon_max],
},
str(accum_file)
)
# Extract if ZIP
extracted = extract_netcdf(accum_file)
if len(extracted) > 0:
accum_file = extracted[0]
# Merge both datasets
print(f" → Merging datasets...")
merge_datasets(instant_file, accum_file, output_file)
size_kb = output_file.stat().st_size / 1024
print(f"✓ {year} - Downloaded ({size_kb:.1f} KB)")
except Exception as e:
print(f"✗ {year} - Failed: {e}")
failed_years.append(year)
# Cleanup partial files
for f in [instant_file, accum_file]:
if f.exists():
f.unlink()
continue
print("\n" + "="*80)
print("DOWNLOAD COMPLETE")
print("="*80)
print(f"Successful: {len(years) - len(failed_years)}/{len(years)} years")
if failed_years:
print(f"\nFailed years: {failed_years}")
print("You can re-run this script to retry failed downloads")
else:
print("\n✓ All years downloaded successfully!")
print(f"\nFiles saved to: {output_dir}")
print("Total files:", len(list(output_dir.glob('era5_*.nc'))))
|