Update app.py

app.py

@@ -41,7 +41,7 @@ import tropycal.tracks as tracks
 # Configuration and Setup
 # -----------------------------
 logging.basicConfig(
-    level=logging.INFO,  # Change to DEBUG for more details
+    level=logging.INFO,  # Use DEBUG for more details
     format='%(asctime)s - %(levelname)s - %(message)s'
 )
 
@@ -53,9 +53,9 @@ DATA_PATH = args.data_path
 # Data paths
 ONI_DATA_PATH = os.path.join(DATA_PATH, 'oni_data.csv')
 TYPHOON_DATA_PATH = os.path.join(DATA_PATH, 'processed_typhoon_data.csv')
-MERGED_DATA_CSV = os.path.join(DATA_PATH, 'merged_typhoon_era5_data.csv')  # Used by other analyses
+MERGED_DATA_CSV = os.path.join(DATA_PATH, 'merged_typhoon_era5_data.csv')  # used in other tabs
 
-# IBTrACS settings (used only for typhoon option updates)
+# IBTrACS settings (only used for updating typhoon options)
 BASIN_FILES = {
     'EP': 'ibtracs.EP.list.v04r01.csv',
     'NA': 'ibtracs.NA.list.v04r01.csv',
@@ -137,6 +137,7 @@ def convert_oni_ascii_to_csv(input_file, output_file):
                         year = str(int(year)-1)
                     data[year][month-1] = anom
     with open(output_file, 'w', newline='') as f:
+        writer = pd.ExcelWriter(f)
         writer = csv.writer(f)
         writer.writerow(['Year','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'])
         for year in sorted(data.keys()):
@@ -155,11 +156,18 @@ def update_oni_data():
             os.remove(temp_file)
 
 def load_data(oni_path, typhoon_path):
-    oni_data = pd.read_csv(oni_path)
-    typhoon_data = pd.read_csv(typhoon_path, low_memory=False)
-    typhoon_data['ISO_TIME'] = pd.to_datetime(typhoon_data['ISO_TIME'], errors='coerce')
-    typhoon_data = typhoon_data.dropna(subset=['ISO_TIME'])
-    return oni_data, typhoon_data
+    if not os.path.exists(typhoon_path):
+        logging.error(f"Typhoon data file not found: {typhoon_path}")
+        return pd.DataFrame(), pd.DataFrame()
+    try:
+        oni_data = pd.read_csv(oni_path)
+        typhoon_data = pd.read_csv(typhoon_path, low_memory=False)
+        typhoon_data['ISO_TIME'] = pd.to_datetime(typhoon_data['ISO_TIME'], errors='coerce')
+        typhoon_data = typhoon_data.dropna(subset=['ISO_TIME'])
+        return oni_data, typhoon_data
+    except Exception as e:
+        logging.error(f"Error loading data: {e}")
+        return pd.DataFrame(), pd.DataFrame()
 
 def process_oni_data(oni_data):
     oni_long = oni_data.melt(id_vars=['Year'], var_name='Month', value_name='ONI')
@@ -366,7 +374,7 @@ def generate_main_analysis(start_year, start_month, end_year, end_month, enso_ph
     filtered_data = merged_data[(merged_data['ISO_TIME']>=start_date) & (merged_data['ISO_TIME']<=end_date)].copy()
     filtered_data['ENSO_Phase'] = filtered_data['ONI'].apply(classify_enso_phases)
     if enso_phase != 'all':
-        filtered_data = filtered_data[filtered_data['ENSO_Phase']==enso_phase.capitalize()]
+        filtered_data = filtered_data[filtered_data['ENSO_Phase'] == enso_phase.capitalize()]
     tracks_fig = generate_typhoon_tracks(filtered_data, typhoon_search)
     wind_scatter = generate_wind_oni_scatter(filtered_data, typhoon_search)
     pressure_scatter = generate_pressure_oni_scatter(filtered_data, typhoon_search)
@@ -379,7 +387,7 @@ def get_full_tracks(start_year, start_month, end_year, end_month, enso_phase, ty
     filtered_data = merged_data[(merged_data['ISO_TIME']>=start_date) & (merged_data['ISO_TIME']<=end_date)].copy()
     filtered_data['ENSO_Phase'] = filtered_data['ONI'].apply(classify_enso_phases)
     if enso_phase != 'all':
-        filtered_data = filtered_data[filtered_data['ENSO_Phase']==enso_phase.capitalize()]
+        filtered_data = filtered_data[filtered_data['ENSO_Phase'] == enso_phase.capitalize()]
     unique_storms = filtered_data['SID'].unique()
     count = len(unique_storms)
     fig = go.Figure()
@@ -468,10 +476,10 @@ def categorize_typhoon_by_standard(wind_speed, standard='atlantic'):
             return 'Tropical Storm', atlantic_standard['Tropical Storm']['hex']
         return 'Tropical Depression', atlantic_standard['Tropical Depression']['hex']
 
-# ------------- Updated TSNE Cluster Function with Mean Path and Stats -------------
+# ------------- Updated TSNE Cluster Function with Mean Curves -------------
 def update_route_clusters(start_year, start_month, end_year, end_month, enso_value, season):
     try:
-        # Use raw typhoon data merged with ONI info so each storm has multiple points.
+        # Merge raw typhoon data with ONI so that each storm has multiple points.
         raw_data = typhoon_data.copy()
         raw_data['Year'] = raw_data['ISO_TIME'].dt.year
         raw_data['Month'] = raw_data['ISO_TIME'].dt.strftime('%m')
@@ -489,7 +497,7 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
             merged_raw = merged_raw[merged_raw['ENSO_Phase'] == enso_value.capitalize()]
         logging.info(f"Total points after ENSO filtering: {merged_raw.shape[0]}")
         
-        # Apply regional filter for Western Pacific (adjust as needed)
+        # Apply regional filter for Western Pacific (adjust boundaries as needed)
         wp_data = merged_raw[(merged_raw['LON'] >= 100) & (merged_raw['LON'] <= 180) &
                              (merged_raw['LAT'] >= 0) & (merged_raw['LAT'] <= 40)]
         logging.info(f"Total points after WP regional filtering: {wp_data.shape[0]}")
@@ -497,7 +505,7 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
             logging.info("WP regional filter returned no data; using all filtered data.")
             wp_data = merged_raw
         
-        # Group by SID so that each storm has multiple points
+        # Group by storm ID (SID); each group must have at least 2 observations
         all_storms_data = []
         for sid, group in wp_data.groupby('SID'):
             group = group.sort_values('ISO_TIME')
@@ -506,16 +514,21 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
             lons = group['LON'].astype(float).values
             if len(lons) < 2:
                 continue
-            all_storms_data.append((sid, lons, lats, times))
+            # Also store wind and pressure for interpolation
+            wind = group['USA_WIND'].astype(float).values if 'USA_WIND' in group.columns else None
+            pres = group['USA_PRES'].astype(float).values if 'USA_PRES' in group.columns else None
+            all_storms_data.append((sid, lons, lats, times, wind, pres))
         logging.info(f"Storms available for TSNE after grouping: {len(all_storms_data)}")
         if not all_storms_data:
             return go.Figure(), go.Figure(), make_subplots(rows=2, cols=1), "No valid storms for clustering."
         
-        # Interpolate each storm route to a common length
+        # Interpolate each storm's route (and wind/pressure) to a common length
         max_length = max(len(item[1]) for item in all_storms_data)
         route_vectors = []
+        wind_curves = []
+        pres_curves = []
         storm_ids = []
-        for sid, lons, lats, times in all_storms_data:
+        for sid, lons, lats, times, wind, pres in all_storms_data:
             t = np.linspace(0, 1, len(lons))
             t_new = np.linspace(0, 1, max_length)
             try:
@@ -529,22 +542,61 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
                 continue
             route_vectors.append(route_vector)
             storm_ids.append(sid)
+            # Interpolate wind and pressure if available; otherwise, fill with NaN
+            if wind is not None and len(wind) >= 2:
+                try:
+                    wind_interp = interp1d(t, wind, kind='linear', fill_value='extrapolate')(t_new)
+                except Exception as ex:
+                    logging.error(f"Wind interpolation error for storm {sid}: {ex}")
+                    wind_interp = np.full(max_length, np.nan)
+            else:
+                wind_interp = np.full(max_length, np.nan)
+            if pres is not None and len(pres) >= 2:
+                try:
+                    pres_interp = interp1d(t, pres, kind='linear', fill_value='extrapolate')(t_new)
+                except Exception as ex:
+                    logging.error(f"Pressure interpolation error for storm {sid}: {ex}")
+                    pres_interp = np.full(max_length, np.nan)
+            else:
+                pres_interp = np.full(max_length, np.nan)
+            wind_curves.append(wind_interp)
+            pres_curves.append(pres_interp)
         logging.info(f"Storms with valid route vectors: {len(route_vectors)}")
         if len(route_vectors) == 0:
             return go.Figure(), go.Figure(), make_subplots(rows=2, cols=1), "No valid storms after interpolation."
         
         route_vectors = np.array(route_vectors)
+        wind_curves = np.array(wind_curves)
+        pres_curves = np.array(pres_curves)
+        
+        # Run TSNE on route vectors
         tsne = TSNE(n_components=2, random_state=42, verbose=1)
         tsne_results = tsne.fit_transform(route_vectors)
         
-        dbscan = DBSCAN(eps=5, min_samples=3)
-        labels = dbscan.fit_predict(tsne_results)
-        unique_labels = sorted(set(labels) - {-1})
+        # Dynamic DBSCAN: choose eps so that we have roughly 5 to 20 clusters if possible
+        selected_labels = None
+        selected_eps = None
+        for eps in np.linspace(1.0, 10.0, 91):
+            dbscan = DBSCAN(eps=eps, min_samples=3)
+            labels = dbscan.fit_predict(tsne_results)
+            clusters = set(labels) - {-1}
+            num_clusters = len(clusters)
+            if 5 <= num_clusters <= 20:
+                selected_labels = labels
+                selected_eps = eps
+                break
+        if selected_labels is None:
+            selected_eps = 5.0
+            dbscan = DBSCAN(eps=selected_eps, min_samples=3)
+            selected_labels = dbscan.fit_predict(tsne_results)
+        logging.info(f"Selected DBSCAN eps: {selected_eps:.2f} yielding {len(set(selected_labels) - {-1})} clusters.")
         
+        # TSNE scatter plot
         fig_tsne = go.Figure()
         colors = px.colors.qualitative.Safe
+        unique_labels = sorted(set(selected_labels) - {-1})
         for i, label in enumerate(unique_labels):
-            indices = np.where(labels == label)[0]
+            indices = np.where(selected_labels == label)[0]
             fig_tsne.add_trace(go.Scatter(
                 x=tsne_results[indices, 0],
                 y=tsne_results[indices, 1],
@@ -552,7 +604,7 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
                 marker=dict(color=colors[i % len(colors)]),
                 name=f"Cluster {label}"
             ))
-        noise_indices = np.where(labels == -1)[0]
+        noise_indices = np.where(selected_labels == -1)[0]
         if len(noise_indices) > 0:
             fig_tsne.add_trace(go.Scatter(
                 x=tsne_results[noise_indices, 0],
@@ -567,12 +619,11 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
             yaxis_title="t-SNE Dim 2"
         )
         
-        # Compute mean route for each cluster and cluster stats (average wind and pressure)
+        # For each cluster, compute mean route, mean wind curve, and mean pressure curve.
         fig_routes = go.Figure()
-        fig_stats = make_subplots(rows=2, cols=1, shared_xaxes=True,
-                                  subplot_titles=("Average Wind Speed (knots)", "Average MSLP (hPa)"))
+        cluster_stats = []  # To hold mean curves for wind and pressure
         for i, label in enumerate(unique_labels):
-            indices = np.where(labels == label)[0]
+            indices = np.where(selected_labels == label)[0]
             cluster_ids = [storm_ids[j] for j in indices]
             cluster_vectors = route_vectors[indices, :]
             mean_vector = np.mean(cluster_vectors, axis=0)
@@ -586,35 +637,51 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
                 line=dict(width=4, color=colors[i % len(colors)]),
                 name=f"Cluster {label} Mean Route"
             ))
-            # Get cluster data from raw filtered data (wp_data)
-            cluster_data = wp_data[wp_data['SID'].isin(cluster_ids)]
-            avg_wind = cluster_data['USA_WIND'].mean() if 'USA_WIND' in cluster_data.columns else np.nan
-            avg_pres = cluster_data['USA_PRES'].mean() if 'USA_PRES' in cluster_data.columns else np.nan
-            # Plot horizontal lines for cluster stats at index i
+            # Get storms in this cluster from wp_data by SID
+            cluster_raw = wp_data[wp_data['SID'].isin(cluster_ids)]
+            # For each storm in the cluster, we already interpolated wind_curves and pres_curves.
+            cluster_winds = wind_curves[indices, :]  # shape: (#storms, max_length)
+            cluster_pres = pres_curves[indices, :]    # shape: (#storms, max_length)
+            # Compute mean curves (if available)
+            if cluster_winds.size > 0:
+                mean_wind_curve = np.nanmean(cluster_winds, axis=0)
+            else:
+                mean_wind_curve = np.full(max_length, np.nan)
+            if cluster_pres.size > 0:
+                mean_pres_curve = np.nanmean(cluster_pres, axis=0)
+            else:
+                mean_pres_curve = np.full(max_length, np.nan)
+            cluster_stats.append((label, mean_wind_curve, mean_pres_curve))
+        
+        # Create cluster stats plot with curves vs normalized route index (0 to 1)
+        x_axis = np.linspace(0, 1, max_length)
+        fig_stats = make_subplots(rows=2, cols=1, shared_xaxes=True,
+                                  subplot_titles=("Mean Wind Speed (knots)", "Mean MSLP (hPa)"))
+        for i, (label, wind_curve, pres_curve) in enumerate(cluster_stats):
             fig_stats.add_trace(go.Scatter(
-                x=[i, i],
-                y=[avg_wind, avg_wind],
-                mode='lines+markers',
+                x=x_axis,
+                y=wind_curve,
+                mode='lines',
                 line=dict(width=2, color=colors[i % len(colors)]),
-                name=f"Cluster {label} Avg Wind"
+                name=f"Cluster {label} Mean Wind"
             ), row=1, col=1)
             fig_stats.add_trace(go.Scatter(
-                x=[i, i],
-                y=[avg_pres, avg_pres],
-                mode='lines+markers',
+                x=x_axis,
+                y=pres_curve,
+                mode='lines',
                 line=dict(width=2, color=colors[i % len(colors)]),
-                name=f"Cluster {label} Avg Pres"
+                name=f"Cluster {label} Mean MSLP"
             ), row=2, col=1)
-        
         fig_stats.update_layout(
-            title="Cluster Statistics",
-            xaxis_title="Cluster Index",
-            yaxis_title="Average Wind Speed (knots)",
-            xaxis2_title="Cluster Index",
-            yaxis2_title="Average MSLP (hPa)",
+            title="Cluster Mean Curves",
+            xaxis_title="Normalized Route Index",
+            yaxis_title="Mean Wind Speed (knots)",
+            xaxis2_title="Normalized Route Index",
+            yaxis2_title="Mean MSLP (hPa)",
             showlegend=True
         )
-        info = "TSNE clustering complete."
+        
+        info = f"TSNE clustering complete. Selected eps: {selected_eps:.2f}. Clusters: {len(unique_labels)}."
         return fig_tsne, fig_routes, fig_stats, info
     except Exception as e:
         logging.error(f"Error in TSNE clustering: {e}")
@@ -784,7 +851,9 @@ with gr.Blocks(title="Typhoon Analysis Dashboard") as demo:
         - **Pressure Analysis**: Analyze pressure vs ONI relationships.
         - **Longitude Analysis**: Study typhoon generation longitude vs ONI.
         - **Path Animation**: View animated storm tracks on a free stock world map (centered at 180°) with a dynamic sidebar and persistent legend.
-        - **TSNE Cluster**: Perform t-SNE clustering on WP storm routes using raw merged typhoon+ONI data with detailed error management. Mean storm routes and cluster-level average wind/pressure are computed.
+        - **TSNE Cluster**: Perform t-SNE clustering on WP storm routes using raw merged typhoon+ONI data.
+          For each cluster, a mean route is computed and, importantly, mean wind and MSLP curves (plotted versus normalized route index)
+          are computed from start to end.
         """)
     
     with gr.Tab("Track Visualization"):