Update app.py

app.py

@@ -6,7 +6,7 @@ import threading
 import time
 from datetime import datetime, timedelta
 from collections import defaultdict
-import csv 
+
 import gradio as gr
 import pandas as pd
 import numpy as np
@@ -41,7 +41,7 @@ import tropycal.tracks as tracks
 # Configuration and Setup
 # -----------------------------
 logging.basicConfig(
-    level=logging.INFO,  # Use DEBUG for more details
+    level=logging.INFO,
     format='%(asctime)s - %(levelname)s - %(message)s'
 )
 
@@ -53,9 +53,9 @@ DATA_PATH = args.data_path
 # Data paths
 ONI_DATA_PATH = os.path.join(DATA_PATH, 'oni_data.csv')
 TYPHOON_DATA_PATH = os.path.join(DATA_PATH, 'processed_typhoon_data.csv')
-MERGED_DATA_CSV = os.path.join(DATA_PATH, 'merged_typhoon_era5_data.csv')  # used in other tabs
+MERGED_DATA_CSV = os.path.join(DATA_PATH, 'merged_typhoon_era5_data.csv')
 
-# IBTrACS settings (only used for updating typhoon options)
+# IBTrACS settings (for typhoon options)
 BASIN_FILES = {
     'EP': 'ibtracs.EP.list.v04r01.csv',
     'NA': 'ibtracs.NA.list.v04r01.csv',
@@ -137,7 +137,6 @@ def convert_oni_ascii_to_csv(input_file, output_file):
                         year = str(int(year)-1)
                     data[year][month-1] = anom
     with open(output_file, 'w', newline='') as f:
-        writer = pd.ExcelWriter(f)
         writer = csv.writer(f)
         writer.writerow(['Year','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'])
         for year in sorted(data.keys()):
@@ -222,7 +221,9 @@ def classify_enso_phases(oni_value):
     else:
         return 'Neutral'
 
-# ------------- Regression Functions -------------
+# -----------------------------
+# Regression Functions
+# -----------------------------
 def perform_wind_regression(start_year, start_month, end_year, end_month):
     start_date = datetime(start_year, start_month, 1)
     end_date = datetime(end_year, end_month, 28)
@@ -262,7 +263,9 @@ def perform_longitude_regression(start_year, start_month, end_year, end_month):
     p_value = model.pvalues['ONI']
     return f"Longitude Regression: β1={beta_1:.4f}, Odds Ratio={exp_beta_1:.4f}, P-value={p_value:.4f}"
 
-# ------------- IBTrACS Data Loading -------------
+# -----------------------------
+# IBTrACS Data Loading
+# -----------------------------
 def load_ibtracs_data():
     ibtracs_data = {}
     for basin, filename in BASIN_FILES.items():
@@ -286,14 +289,18 @@ def load_ibtracs_data():
 
 ibtracs = load_ibtracs_data()
 
-# ------------- Load & Process Data -------------
+# -----------------------------
+# Load & Process Data
+# -----------------------------
 update_oni_data()
 oni_data, typhoon_data = load_data(ONI_DATA_PATH, TYPHOON_DATA_PATH)
 oni_long = process_oni_data(oni_data)
 typhoon_max = process_typhoon_data(typhoon_data)
 merged_data = merge_data(oni_long, typhoon_max)
 
-# ------------- Visualization Functions -------------
+# -----------------------------
+# Visualization Functions
+# -----------------------------
 def generate_typhoon_tracks(filtered_data, typhoon_search):
     fig = go.Figure()
     for sid in filtered_data['SID'].unique():
@@ -394,11 +401,12 @@ def get_full_tracks(start_year, start_month, end_year, end_month, enso_phase, ty
     for sid in unique_storms:
         storm_data = typhoon_data[typhoon_data['SID']==sid]
         name = storm_data['NAME'].iloc[0] if pd.notnull(storm_data['NAME'].iloc[0]) else "Unnamed"
+        basin = storm_data['SID'].iloc[0][:2]  # First 2 characters often denote basin
         storm_oni = filtered_data[filtered_data['SID']==sid]['ONI'].iloc[0]
         color = 'red' if storm_oni>=0.5 else ('blue' if storm_oni<=-0.5 else 'green')
         fig.add_trace(go.Scattergeo(
             lon=storm_data['LON'], lat=storm_data['LAT'], mode='lines',
-            name=f"{name} ({storm_data['SEASON'].iloc[0]})",
+            name=f"{name} ({basin})",
             line=dict(width=1.5, color=color), hoverinfo="name"
         ))
     if typhoon_search:
@@ -408,7 +416,7 @@ def get_full_tracks(start_year, start_month, end_year, end_month, enso_phase, ty
                 storm_data = typhoon_data[typhoon_data['SID']==sid]
                 fig.add_trace(go.Scattergeo(
                     lon=storm_data['LON'], lat=storm_data['LAT'], mode='lines+markers',
-                    name=f"MATCHED: {storm_data['NAME'].iloc[0]} ({storm_data['SEASON'].iloc[0]})",
+                    name=f"MATCHED: {storm_data['NAME'].iloc[0]}",
                     line=dict(width=3, color='yellow'),
                     marker=dict(size=5), hoverinfo="name"
                 ))
@@ -430,7 +438,7 @@ def get_full_tracks(start_year, start_month, end_year, end_month, enso_phase, ty
     )
     fig.add_annotation(
         x=0.02, y=0.98, xref="paper", yref="paper",
-        text="Red: El Niño, Blue: La Niña, Green: Neutral",
+        text="Red: El Niño, Blue: La Nina, Green: Neutral",
         showarrow=False, align="left",
         bgcolor="rgba(255,255,255,0.8)"
     )
@@ -476,10 +484,12 @@ def categorize_typhoon_by_standard(wind_speed, standard='atlantic'):
             return 'Tropical Storm', atlantic_standard['Tropical Storm']['hex']
         return 'Tropical Depression', atlantic_standard['Tropical Depression']['hex']
 
-# ------------- Updated TSNE Cluster Function with Mean Curves -------------
+# -----------------------------
+# Updated TSNE Cluster Function with Mean Curves
+# -----------------------------
 def update_route_clusters(start_year, start_month, end_year, end_month, enso_value, season):
     try:
-        # Merge raw typhoon data with ONI so that each storm has multiple points.
+        # Merge raw typhoon data with ONI so each storm has multiple observations.
         raw_data = typhoon_data.copy()
         raw_data['Year'] = raw_data['ISO_TIME'].dt.year
         raw_data['Month'] = raw_data['ISO_TIME'].dt.strftime('%m')
@@ -497,7 +507,7 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
             merged_raw = merged_raw[merged_raw['ENSO_Phase'] == enso_value.capitalize()]
         logging.info(f"Total points after ENSO filtering: {merged_raw.shape[0]}")
         
-        # Apply regional filter for Western Pacific (adjust boundaries as needed)
+        # Regional filtering for Western Pacific
         wp_data = merged_raw[(merged_raw['LON'] >= 100) & (merged_raw['LON'] <= 180) &
                              (merged_raw['LAT'] >= 0) & (merged_raw['LAT'] <= 40)]
         logging.info(f"Total points after WP regional filtering: {wp_data.shape[0]}")
@@ -505,7 +515,7 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
             logging.info("WP regional filter returned no data; using all filtered data.")
             wp_data = merged_raw
         
-        # Group by storm ID (SID); each group must have at least 2 observations
+        # Group by storm ID so each storm has multiple observations
         all_storms_data = []
         for sid, group in wp_data.groupby('SID'):
             group = group.sort_values('ISO_TIME')
@@ -514,7 +524,7 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
             lons = group['LON'].astype(float).values
             if len(lons) < 2:
                 continue
-            # Also store wind and pressure for interpolation
+            # Also extract wind and pressure curves
             wind = group['USA_WIND'].astype(float).values if 'USA_WIND' in group.columns else None
             pres = group['USA_PRES'].astype(float).values if 'USA_PRES' in group.columns else None
             all_storms_data.append((sid, lons, lats, times, wind, pres))
@@ -522,7 +532,7 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
         if not all_storms_data:
             return go.Figure(), go.Figure(), make_subplots(rows=2, cols=1), "No valid storms for clustering."
         
-        # Interpolate each storm's route (and wind/pressure) to a common length
+        # Interpolate each storm's route, wind, and pressure to a common length
         max_length = max(len(item[1]) for item in all_storms_data)
         route_vectors = []
         wind_curves = []
@@ -542,7 +552,7 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
                 continue
             route_vectors.append(route_vector)
             storm_ids.append(sid)
-            # Interpolate wind and pressure if available; otherwise, fill with NaN
+            # Interpolate wind and pressure if available
             if wind is not None and len(wind) >= 2:
                 try:
                     wind_interp = interp1d(t, wind, kind='linear', fill_value='extrapolate')(t_new)
@@ -573,15 +583,14 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
         tsne = TSNE(n_components=2, random_state=42, verbose=1)
         tsne_results = tsne.fit_transform(route_vectors)
         
-        # Dynamic DBSCAN: choose eps so that we have roughly 5 to 20 clusters if possible
+        # Dynamic DBSCAN: choose eps to yield roughly 5 to 20 clusters
         selected_labels = None
         selected_eps = None
         for eps in np.linspace(1.0, 10.0, 91):
             dbscan = DBSCAN(eps=eps, min_samples=3)
             labels = dbscan.fit_predict(tsne_results)
             clusters = set(labels) - {-1}
-            num_clusters = len(clusters)
-            if 5 <= num_clusters <= 20:
+            if 5 <= len(clusters) <= 20:
                 selected_labels = labels
                 selected_eps = eps
                 break
@@ -589,7 +598,7 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
             selected_eps = 5.0
             dbscan = DBSCAN(eps=selected_eps, min_samples=3)
             selected_labels = dbscan.fit_predict(tsne_results)
-        logging.info(f"Selected DBSCAN eps: {selected_eps:.2f} yielding {len(set(selected_labels) - {-1})} clusters.")
+        logging.info(f"Selected DBSCAN eps: {selected_eps:.2f} yielding {len(set(selected_labels)-{-1})} clusters.")
         
         # TSNE scatter plot
         fig_tsne = go.Figure()
@@ -619,9 +628,9 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
             yaxis_title="t-SNE Dim 2"
         )
         
-        # For each cluster, compute mean route, mean wind curve, and mean pressure curve.
+        # For each cluster, compute mean route, and compute mean wind and pressure curves along normalized route index.
         fig_routes = go.Figure()
-        cluster_stats = []  # To hold mean curves for wind and pressure
+        cluster_stats = []  # To hold mean curves per cluster
         for i, label in enumerate(unique_labels):
             indices = np.where(selected_labels == label)[0]
             cluster_ids = [storm_ids[j] for j in indices]
@@ -637,23 +646,14 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
                 line=dict(width=4, color=colors[i % len(colors)]),
                 name=f"Cluster {label} Mean Route"
             ))
-            # Get storms in this cluster from wp_data by SID
-            cluster_raw = wp_data[wp_data['SID'].isin(cluster_ids)]
-            # For each storm in the cluster, we already interpolated wind_curves and pres_curves.
-            cluster_winds = wind_curves[indices, :]  # shape: (#storms, max_length)
-            cluster_pres = pres_curves[indices, :]    # shape: (#storms, max_length)
-            # Compute mean curves (if available)
-            if cluster_winds.size > 0:
-                mean_wind_curve = np.nanmean(cluster_winds, axis=0)
-            else:
-                mean_wind_curve = np.full(max_length, np.nan)
-            if cluster_pres.size > 0:
-                mean_pres_curve = np.nanmean(cluster_pres, axis=0)
-            else:
-                mean_pres_curve = np.full(max_length, np.nan)
+            # Retrieve raw wind and pressure curves for storms in this cluster
+            cluster_winds = wind_curves[indices, :]
+            cluster_pres = pres_curves[indices, :]
+            mean_wind_curve = np.nanmean(cluster_winds, axis=0)
+            mean_pres_curve = np.nanmean(cluster_pres, axis=0)
             cluster_stats.append((label, mean_wind_curve, mean_pres_curve))
         
-        # Create cluster stats plot with curves vs normalized route index (0 to 1)
+        # Create a cluster stats plot with curves vs normalized route index (0 to 1)
         x_axis = np.linspace(0, 1, max_length)
         fig_stats = make_subplots(rows=2, cols=1, shared_xaxes=True,
                                   subplot_titles=("Mean Wind Speed (knots)", "Mean MSLP (hPa)"))
@@ -687,7 +687,9 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
         logging.error(f"Error in TSNE clustering: {e}")
         return go.Figure(), go.Figure(), make_subplots(rows=2, cols=1), f"Error in TSNE clustering: {e}"
 
-# ------------- Animation Functions Using Processed CSV & Stock Map -------------
+# -----------------------------
+# Animation Functions Using Processed CSV & Stock Map
+# -----------------------------
 def generate_track_video_from_csv(year, storm_id, standard):
     storm_df = typhoon_data[typhoon_data['SID'] == storm_id].copy()
     if storm_df.empty:
@@ -702,6 +704,7 @@ def generate_track_video_from_csv(year, storm_id, standard):
     else:
         winds = np.full(len(lats), np.nan)
     storm_name = storm_df['NAME'].iloc[0]
+    basin = storm_df['SID'].iloc[0][:2]  # Use first 2 characters as basin code
     season = storm_df['SEASON'].iloc[0]
     
     min_lat, max_lat = np.min(lats), np.max(lats)
@@ -718,12 +721,13 @@ def generate_track_video_from_csv(year, storm_id, standard):
     ax.coastlines(resolution='50m', color='black', linewidth=1)
     gl = ax.gridlines(draw_labels=True, color='gray', alpha=0.4, linestyle='--')
     gl.top_labels = gl.right_labels = False
-    ax.set_title(f"{year} {storm_name} - {season}", fontsize=14)
+    ax.set_title(f"{year} {storm_name} ({basin}) - {season}", fontsize=14)
     
     line, = ax.plot([], [], transform=ccrs.PlateCarree(), color='blue', linewidth=2)
     point, = ax.plot([], [], 'o', markersize=8, transform=ccrs.PlateCarree())
     date_text = ax.text(0.02, 0.02, '', transform=ax.transAxes, fontsize=10,
                         bbox=dict(facecolor='white', alpha=0.8))
+    # Display storm name and basin in a dynamic sidebar
     storm_info_text = fig.text(0.70, 0.60, '', fontsize=10,
                                bbox=dict(facecolor='white', alpha=0.8, boxstyle='round,pad=0.5'))
     
@@ -750,10 +754,7 @@ def generate_track_video_from_csv(year, storm_id, standard):
         point.set_color(color)
         dt_str = pd.to_datetime(times[frame]).strftime('%Y-%m-%d %H:%M')
         date_text.set_text(dt_str)
-        info_str = (f"Name: {storm_name}\n"
-                    f"Date: {dt_str}\n"
-                    f"Wind: {wind_speed:.1f} kt\n"
-                    f"Category: {category}")
+        info_str = (f"Name: {storm_name}\nBasin: {basin}\nDate: {dt_str}\nWind: {wind_speed:.1f} kt\nCategory: {category}")
         storm_info_text.set_text(info_str)
         return line, point, date_text, storm_info_text
 
@@ -771,7 +772,9 @@ def simplified_track_video(year, basin, typhoon, standard):
     storm_id = typhoon.split('(')[-1].strip(')')
     return generate_track_video_from_csv(year, storm_id, standard)
 
-# ------------- Typhoon Options Update Functions -------------
+# -----------------------------
+# Typhoon Options Update Functions
+# -----------------------------
 basin_to_prefix = {
     "All Basins": "all",
     "NA - North Atlantic": "NA",
@@ -835,7 +838,9 @@ def update_typhoon_options_anim(year, basin):
         logging.error(f"Error in update_typhoon_options_anim: {e}")
         return gr.update(choices=[], value=None)
 
-# ------------- Gradio Interface -------------
+# -----------------------------
+# Gradio Interface
+# -----------------------------
 with gr.Blocks(title="Typhoon Analysis Dashboard") as demo:
     gr.Markdown("# Typhoon Analysis Dashboard")
     
@@ -850,10 +855,9 @@ with gr.Blocks(title="Typhoon Analysis Dashboard") as demo:
         - **Wind Analysis**: Examine wind speed vs ONI relationships.
         - **Pressure Analysis**: Analyze pressure vs ONI relationships.
         - **Longitude Analysis**: Study typhoon generation longitude vs ONI.
-        - **Path Animation**: View animated storm tracks on a free stock world map (centered at 180°) with a dynamic sidebar and persistent legend.
-        - **TSNE Cluster**: Perform t-SNE clustering on WP storm routes using raw merged typhoon+ONI data.
-          For each cluster, a mean route is computed and, importantly, mean wind and MSLP curves (plotted versus normalized route index)
-          are computed from start to end.
+        - **Path Animation**: View animated storm tracks on a free stock world map (centered at 180°) with a dynamic sidebar that shows the typhoon name and basin.
+        - **TSNE Cluster**: Perform t-SNE clustering on WP storm routes using raw merged typhoon+ONI data with detailed error management.
+          Mean routes and evolving curves (wind and pressure vs. normalized route index) are computed.
         """)
     
     with gr.Tab("Track Visualization"):
@@ -918,9 +922,10 @@ with gr.Blocks(title="Typhoon Analysis Dashboard") as demo:
                               outputs=[regression_plot, slopes_text, lon_regression_results])
     
     with gr.Tab("Tropical Cyclone Path Animation"):
+        # Basin selection removed. Always use All Basins.
         with gr.Row():
             year_dropdown = gr.Dropdown(label="Year", choices=[str(y) for y in range(1950,2025)], value="2000")
-            basin_dropdown = gr.Dropdown(label="Basin", choices=["NA - North Atlantic","EP - Eastern North Pacific","WP - Western North Pacific","All Basins"], value="NA - North Atlantic")
+            # Remove basin dropdown and set it internally to "all"
         with gr.Row():
             typhoon_dropdown = gr.Dropdown(label="Tropical Cyclone")
             standard_dropdown = gr.Dropdown(label="Classification Standard", choices=['atlantic','taiwan'], value='atlantic')
@@ -928,16 +933,16 @@ with gr.Blocks(title="Typhoon Analysis Dashboard") as demo:
         path_video = gr.Video(label="Tropical Cyclone Path Animation", format="mp4", interactive=False, elem_id="path_video")
         animation_info = gr.Markdown("""
         ### Animation Instructions
-        1. Select a year and basin (data is from your processed CSV).
+        1. Select a year (data is from your processed CSV, using all basins).
         2. Choose a tropical cyclone from the populated list.
         3. Select a classification standard (Atlantic or Taiwan).
         4. Click "Generate Animation".
-        5. The animation displays the storm track on a free stock world map (centered at 180°) with a dynamic sidebar and persistent legend.
+        5. The animation displays the storm track on a free stock world map (centered at 180°) with a dynamic sidebar.
+           The sidebar shows the storm name and basin.
         """)
-        year_dropdown.change(fn=update_typhoon_options_anim, inputs=[year_dropdown, basin_dropdown], outputs=typhoon_dropdown)
-        basin_dropdown.change(fn=update_typhoon_options_anim, inputs=[year_dropdown, basin_dropdown], outputs=typhoon_dropdown)
+        year_dropdown.change(fn=update_typhoon_options_anim, inputs=[year_dropdown, gr.State("dummy")], outputs=typhoon_dropdown)
         animate_btn.click(fn=simplified_track_video,
-                          inputs=[year_dropdown, basin_dropdown, typhoon_dropdown, standard_dropdown],
+                          inputs=[year_dropdown, gr.Textbox.update(value="All Basins"), typhoon_dropdown, standard_dropdown],
                           outputs=path_video)
     
     with gr.Tab("TSNE Cluster"):