Update app.py

app.py

@@ -33,8 +33,16 @@ DATA_PATH = args.data_path
 
 ONI_DATA_PATH = os.path.join(DATA_PATH, 'oni_data.csv')
 TYPHOON_DATA_PATH = os.path.join(DATA_PATH, 'processed_typhoon_data.csv')
-LOCAL_iBtrace_PATH = os.path.join(DATA_PATH, 'ibtracs.ALL.list.v04r01.csv') 
-iBtrace_uri = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r01/access/csv/ibtracs.ALL.list.v04r01.csv'
+
+# Basin-specific files instead of the global file
+BASIN_FILES = {
+    'EP': 'ibtracs.EP.list.v04r01.csv',
+    'NA': 'ibtracs.NA.list.v04r01.csv',
+    'WP': 'ibtracs.WP.list.v04r01.csv'
+}
+IBTRACS_BASE_URL = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r01/access/csv/'
+LOCAL_MERGED_PATH = os.path.join(DATA_PATH, 'ibtracs.merged.v04r01.csv')
+
 CACHE_FILE = 'ibtracs_cache.pkl'
 CACHE_EXPIRY_DAYS = 0  # Force refresh for testing
 
@@ -130,21 +138,54 @@ def load_ibtracs_data():
         with open(CACHE_FILE, 'rb') as f:
             return pickle.load(f)
     
-    all_basins_path = os.path.join(DATA_PATH, 'ibtracs.ALL.list.v04r01.csv')
-    
     try:
-        if os.path.exists(all_basins_path):
-            print("Loading ALL basins file...")
-            ibtracs = tracks.TrackDataset(source='ibtracs', ibtracs_url=all_basins_path)
+        # Check if merged file already exists
+        if os.path.exists(LOCAL_MERGED_PATH):
+            print("Loading merged basins file...")
+            ibtracs = tracks.TrackDataset(source='ibtracs', ibtracs_url=LOCAL_MERGED_PATH)
         else:
-            print("Downloading ALL basins file...")
-            response = requests.get(iBtrace_uri)
-            response.raise_for_status()
-            with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.csv') as temp_file:
-                temp_file.write(response.text)
-                shutil.move(temp_file.name, all_basins_path)
-            print(f"Downloaded {all_basins_path}. Verify it contains all basins.")
-            ibtracs = tracks.TrackDataset(source='ibtracs', ibtracs_url=all_basins_path)
+            print("Downloading and merging basin files...")
+            
+            # Create temporary file for merged data
+            header = None
+            with open(LOCAL_MERGED_PATH, 'w', newline='') as merged_file:
+                writer = None
+                
+                # Download and process each basin file
+                for basin, filename in BASIN_FILES.items():
+                    basin_url = IBTRACS_BASE_URL + filename
+                    local_path = os.path.join(DATA_PATH, filename)
+                    
+                    # Download the basin file if it doesn't exist
+                    if not os.path.exists(local_path):
+                        print(f"Downloading {basin} basin file...")
+                        response = requests.get(basin_url)
+                        response.raise_for_status()
+                        with open(local_path, 'wb') as f:
+                            f.write(response.content)
+                        print(f"Downloaded {basin} basin file.")
+                    
+                    # Process and merge the basin file
+                    with open(local_path, 'r', newline='') as basin_file:
+                        reader = csv.reader(basin_file)
+                        
+                        # Save header from the first file
+                        if header is None:
+                            header = next(reader)
+                            writer = csv.writer(merged_file)
+                            writer.writerow(header)
+                            # Skip the second header line
+                            next(reader)
+                        else:
+                            # Skip header lines in subsequent files
+                            next(reader)
+                            next(reader)
+                        
+                        # Write all data rows
+                        writer.writerows(reader)
+            
+            print(f"Created merged basin file at {LOCAL_MERGED_PATH}")
+            ibtracs = tracks.TrackDataset(source='ibtracs', ibtracs_url=LOCAL_MERGED_PATH)
         
         with open(CACHE_FILE, 'wb') as f:
             pickle.dump(ibtracs, f)
@@ -249,7 +290,7 @@ def classify_enso_phases(oni_value):
 # Load data globally
 update_oni_data()
 ibtracs = load_ibtracs_data()
-convert_typhoondata(LOCAL_iBtrace_PATH, TYPHOON_DATA_PATH)
+convert_typhoondata(LOCAL_MERGED_PATH, TYPHOON_DATA_PATH)
 oni_data, typhoon_data = load_data(ONI_DATA_PATH, TYPHOON_DATA_PATH)
 oni_long = process_oni_data(oni_data)
 typhoon_max = process_typhoon_data(typhoon_data)