Update app.py

app.py

@@ -8,6 +8,7 @@ import cartopy.crs as ccrs
 import cartopy.feature as cfeature
 import plotly.graph_objects as go
 import plotly.express as px
+from plotly.subplots import make_subplots
 import tropycal.tracks as tracks
 import pickle
 import requests
@@ -22,6 +23,7 @@ from collections import defaultdict
 import filecmp
 from sklearn.manifold import TSNE
 from sklearn.cluster import DBSCAN
+from scipy.interpolate import interp1d
 
 # Command-line argument parsing
 parser = argparse.ArgumentParser(description='Typhoon Analysis Dashboard')
@@ -72,6 +74,16 @@ season_months = {
     'winter': [12, 1, 2]
 }
 
+# Regions for duration calculations
+regions = {
+    "Taiwan Land": {"lat_min": 21.8, "lat_max": 25.3, "lon_min": 119.5, "lon_max": 122.1},
+    "Taiwan Sea": {"lat_min": 19, "lat_max": 28, "lon_min": 117, "lon_max": 125},
+    "Japan": {"lat_min": 20, "lat_max": 45, "lon_min": 120, "lon_max": 150},
+    "China": {"lat_min": 18, "lat_max": 53, "lon_min": 73, "lon_max": 135},
+    "Hong Kong": {"lat_min": 21.5, "lat_max": 23, "lon_min": 113, "lon_max": 115},
+    "Philippines": {"lat_min": 5, "lat_max": 21, "lon_min": 115, "lon_max": 130}
+}
+
 # Data loading and preprocessing functions
 def download_oni_file(url, filename):
     response = requests.get(url)
@@ -449,10 +461,70 @@ def perform_longitude_regression(start_year, start_month, end_year, end_month):
 
 # t-SNE clustering functions
 def filter_west_pacific_coordinates(lons, lats):
-    mask = (lons >= 100) & (lons <= 180) & (lats >= 0) & (lats <= 50)
+    mask = (lons >= 100) & (lons <= 180) & (lats >= 0) & (lats <= 40)
     return lons[mask], lats[mask]
 
-def dynamic_dbscan(tsne_results, min_clusters=10, max_clusters=20, eps_values=np.arange(0.1, 5.0, 0.1)):
+def filter_storm_by_season(storm, season):
+    start_month = storm.time[0].month
+    if season == 'all':
+        return True
+    elif season == 'summer':
+        return 4 <= start_month <= 8
+    elif season == 'winter':
+        return 9 <= start_month <= 12
+    return False
+
+def point_region(lat, lon):
+    twl = regions["Taiwan Land"]
+    if twl["lat_min"] <= lat <= twl["lat_max"] and twl["lon_min"] <= lon <= twl["lon_max"]:
+        return "Taiwan Land"
+    tws = regions["Taiwan Sea"]
+    if tws["lat_min"] <= lat <= tws["lat_max"] and tws["lon_min"] <= lon <= tws["lon_max"]:
+        if not (twl["lat_min"] <= lat <= twl["lat_max"] and twl["lon_min"] <= lon <= twl["lon_max"]):
+            return "Taiwan Sea"
+    for rg in ["Japan", "China", "Hong Kong", "Philippines"]:
+        box = regions[rg]
+        if box["lat_min"] <= lat <= box["lat_max"] and box["lon_min"] <= lon <= box["lon_max"]:
+            return rg
+    return None
+
+def calculate_region_durations(lons, lats, times):
+    region_times = defaultdict(float)
+    point_regions_list = [point_region(lats[i], lons[i]) for i in range(len(lons))]
+    for i in range(len(lons) - 1):
+        dt = (times[i + 1] - times[i]).total_seconds() / 3600.0
+        r1 = point_regions_list[i]
+        r2 = point_regions_list[i + 1]
+        if r1 and r2:
+            if r1 == r2:
+                region_times[r1] += dt
+            else:
+                region_times[r1] += dt / 2
+                region_times[r2] += dt / 2
+        elif r1 and not r2:
+            region_times[r1] += dt / 2
+        elif r2 and not r1:
+            region_times[r2] += dt / 2
+    return dict(region_times)
+
+def endpoint_region_label(cluster_label, cluster_labels, filtered_storms):
+    indices = np.where(cluster_labels == cluster_label)[0]
+    if len(indices) == 0:
+        return ""
+    end_count = defaultdict(int)
+    for idx in indices:
+        lons, lats, vmax_, mslp_, times = filtered_storms[idx]
+        reg = point_region(lats[-1], lons[-1])
+        if reg:
+            end_count[reg] += 1
+    if end_count:
+        max_reg = max(end_count, key=end_count.get)
+        ratio = end_count[max_reg] / len(indices)
+        if ratio > 0.5:
+            return max_reg
+    return ""
+
+def dynamic_dbscan(tsne_results, min_clusters=10, max_clusters=20, eps_values=np.linspace(1.0, 10.0, 91)):
     best_labels = None
     best_n_clusters = 0
     best_n_noise = len(tsne_results)
@@ -460,7 +532,10 @@ def dynamic_dbscan(tsne_results, min_clusters=10, max_clusters=20, eps_values=np
     for eps in eps_values:
         dbscan = DBSCAN(eps=eps, min_samples=3)
         labels = dbscan.fit_predict(tsne_results)
-        n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
+        unique_labels = set(labels)
+        if -1 in unique_labels:
+            unique_labels.remove(-1)
+        n_clusters = len(unique_labels)
         n_noise = np.sum(labels == -1)
         if min_clusters <= n_clusters <= max_clusters and n_noise < best_n_noise:
             best_labels = labels
@@ -468,11 +543,19 @@ def dynamic_dbscan(tsne_results, min_clusters=10, max_clusters=20, eps_values=np
             best_n_noise = n_noise
             best_eps = eps
     if best_labels is None:
-        dbscan = DBSCAN(eps=eps_values[0], min_samples=3)
-        best_labels = dbscan.fit_predict(tsne_results)
-        best_n_clusters = len(set(best_labels)) - (1 if -1 in best_labels else 0)
-        best_n_noise = np.sum(best_labels == -1)
-        best_eps = eps_values[0]
+        for eps in eps_values[::-1]:
+            dbscan = DBSCAN(eps=eps, min_samples=3)
+            labels = dbscan.fit_predict(tsne_results)
+            unique_labels = set(labels)
+            if -1 in unique_labels:
+                unique_labels.remove(-1)
+            n_clusters = len(unique_labels)
+            if n_clusters == max_clusters:
+                best_labels = labels
+                best_n_clusters = n_clusters
+                best_n_noise = np.sum(labels == -1)
+                best_eps = eps
+                break
     return best_labels, best_n_clusters, best_n_noise, best_eps
 
 def update_route_clusters(start_year, start_month, end_year, end_month, enso_value, season):
@@ -484,7 +567,7 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
         season_data = ibtracs.get_season(year)
         for storm_id in season_data.summary()['id']:
             storm = ibtracs.get_storm(storm_id)
-            if storm.time[0] >= start_date and storm.time[-1] <= end_date:
+            if storm.time[0] >= start_date and storm.time[-1] <= end_date and filter_storm_by_season(storm, season):
                 lons, lats = filter_west_pacific_coordinates(np.array(storm.lon), np.array(storm.lat))
                 if len(lons) > 1:
                     start_time = storm.time[0]
@@ -497,79 +580,235 @@ def update_route_clusters(start_year, start_month, end_year, end_month, enso_val
                         if enso_value == 'all' or enso_phase_storm == enso_value.capitalize():
                             all_storms_data.append((lons, lats, np.array(storm.vmax), np.array(storm.mslp), np.array(storm.time), storm.name, enso_phase_storm))
 
-    if season != 'all':
-        all_storms_data = [storm for storm in all_storms_data if storm[4][0].month in season_months[season]]
-
     if not all_storms_data:
-        return go.Figure(), go.Figure(), go.Figure(), "No storms found in the selected period."
+        return go.Figure(), go.Figure(), make_subplots(rows=2, cols=1), "No storms found in the selected period."
 
     # Prepare route vectors for t-SNE
     max_length = max(len(st[0]) for st in all_storms_data)
     route_vectors = []
-    for lons, lats, _, _, _, _, _ in all_storms_data:
-        interp_lons = np.interp(np.linspace(0, 1, max_length), np.linspace(0, 1, len(lons)), lons)
-        interp_lats = np.interp(np.linspace(0, 1, max_length), np.linspace(0, 1, len(lats)), lats)
-        route_vectors.append(np.column_stack((interp_lons, interp_lats)).flatten())
+    filtered_storms = []
+    storms_vmax_list = []
+    storms_mslp_list = []
+    for idx, (lons, lats, vmax, mslp, times, name, enso_phase) in enumerate(all_storms_data):
+        t = np.linspace(0, 1, len(lons))
+        t_new = np.linspace(0, 1, max_length)
+        try:
+            lon_i = interp1d(t, lons, kind='linear', fill_value='extrapolate')(t_new)
+            lat_i = interp1d(t, lats, kind='linear', fill_value='extrapolate')(t_new)
+            vmax_i = interp1d(t, vmax, kind='linear', fill_value='extrapolate')(t_new)
+            mslp_i = interp1d(t, mslp, kind='linear', fill_value='extrapolate')(t_new)
+        except Exception as e:
+            continue
+        route_vector = np.column_stack((lon_i, lat_i)).flatten()
+        if np.isnan(route_vector).any():
+            continue
+        route_vectors.append(route_vector)
+        filtered_storms.append((lons, lats, vmax_i, mslp_i, times))
+        storms_vmax_list.append(vmax_i)
+        storms_mslp_list.append(mslp_i)
+
     route_vectors = np.array(route_vectors)
+    if len(route_vectors) == 0:
+        return go.Figure(), go.Figure(), make_subplots(rows=2, cols=1), "No valid storms after interpolation."
 
     # Perform t-SNE
-    tsne_results = TSNE(n_components=2, random_state=42, perplexity=min(30, len(route_vectors)-1)).fit_transform(route_vectors)
+    tsne = TSNE(n_components=2, random_state=42, verbose=1)
+    tsne_results = tsne.fit_transform(route_vectors)
 
     # Dynamic DBSCAN clustering
     best_labels, best_n_clusters, best_n_noise, best_eps = dynamic_dbscan(tsne_results)
 
+    # Calculate region durations and mean routes
+    unique_labels = sorted(set(best_labels) - {-1})
+    label_to_idx = {label: i for i, label in enumerate(unique_labels)}
+    cluster_region_durations = [defaultdict(float) for _ in range(len(unique_labels))]
+    cluster_mean_routes = []
+    cluster_mean_vmax = []
+    cluster_mean_mslp = []
+
+    for i, (lons, lats, vmax, mslp, times) in enumerate(filtered_storms):
+        c = best_labels[i]
+        if c == -1:
+            continue
+        durations = calculate_region_durations(lons, lats, times)
+        idx = label_to_idx[c]
+        for r, val in durations.items():
+            cluster_region_durations[idx][r] += val
+
+    for c in unique_labels:
+        indices = np.where(best_labels == c)[0]
+        if len(indices) == 0:
+            cluster_mean_routes.append(([], []))
+            cluster_mean_vmax.append([])
+            cluster_mean_mslp.append([])
+            continue
+        cluster_lons = []
+        cluster_lats = []
+        cluster_v = []
+        cluster_p = []
+        for idx in indices:
+            lons, lats, vmax_, mslp_, times = filtered_storms[idx]
+            t = np.linspace(0, 1, len(lons))
+            t_new = np.linspace(0, 1, max_length)
+            lon_i = interp1d(t, lons, kind='linear', fill_value='extrapolate')(t_new)
+            lat_i = interp1d(t, lats, kind='linear', fill_value='extrapolate')(t_new)
+            cluster_lons.append(lon_i)
+            cluster_lats.append(lat_i)
+            cluster_v.append(storms_vmax_list[idx])
+            cluster_p.append(storms_mslp_list[idx])
+        if cluster_lons and cluster_lats:
+            mean_lon = np.mean(cluster_lons, axis=0)
+            mean_lat = np.mean(cluster_lats, axis=0)
+            mean_v = np.mean(cluster_v, axis=0)
+            mean_p = np.mean(cluster_p, axis=0)
+            cluster_mean_routes.append((mean_lon, mean_lat))
+            cluster_mean_vmax.append(mean_v)
+            cluster_mean_mslp.append(mean_p)
+        else:
+            cluster_mean_routes.append(([], []))
+            cluster_mean_vmax.append([])
+            cluster_mean_mslp.append([])
+
     # t-SNE Scatter Plot
     fig_tsne = go.Figure()
-    for cluster in set(best_labels):
-        mask = best_labels == cluster
-        name = "Noise" if cluster == -1 else f"Cluster {cluster}"
+    cluster_colors = px.colors.qualitative.Safe
+    if len(cluster_colors) < len(unique_labels):
+        cluster_colors = px.colors.qualitative.Dark24
+    for i, c in enumerate(unique_labels):
+        indices = np.where(best_labels == c)[0]
+        end_reg = endpoint_region_label(c, best_labels, filtered_storms)
+        name = f"Cluster {i+1}" + (f" (towards {end_reg})" if end_reg else "")
         fig_tsne.add_trace(go.Scatter(
-            x=tsne_results[mask, 0], y=tsne_results[mask, 1], mode='markers',
-            name=name, text=[all_storms_data[i][5] for i in range(len(all_storms_data)) if mask[i]],
-            hoverinfo='text'
+            x=tsne_results[indices, 0],
+            y=tsne_results[indices, 1],
+            mode='markers',
+            marker=dict(size=5, color=cluster_colors[i % len(cluster_colors)]),
+            name=name
         ))
-    fig_tsne.update_layout(title="t-SNE Clustering of Typhoon Routes", xaxis_title="t-SNE 1", yaxis_title="t-SNE 2")
+    noise_indices = np.where(best_labels == -1)[0]
+    if len(noise_indices) > 0:
+        fig_tsne.add_trace(go.Scatter(
+            x=tsne_results[noise_indices, 0],
+            y=tsne_results[noise_indices, 1],
+            mode='markers',
+            marker=dict(size=5, color='grey'),
+            name='Noise'
+        ))
+    fig_tsne.update_layout(
+        title="TSNE of Typhoon Routes",
+        xaxis_title="TSNE Dim 1",
+        yaxis_title="TSNE Dim 2",
+        legend_title="Clusters"
+    )
 
-    # Typhoon Routes Plot
+    # Typhoon Routes Plot with Mean Routes
     fig_routes = go.Figure()
-    for i, (lons, lats, _, _, _, name, _) in enumerate(all_storms_data):
-        cluster = best_labels[i]
-        color = 'gray' if cluster == -1 else px.colors.qualitative.Plotly[cluster % len(px.colors.qualitative.Plotly)]
-        fig_routes.add_trace(go.Scattergeo(
-            lon=lons, lat=lats, mode='lines+markers', name=name,
-            line=dict(color=color), marker=dict(size=4), hoverinfo='text', text=name
-        ))
+    for i, (lons, lats, _, _, _) in enumerate(filtered_storms):
+        c = best_labels[i]
+        if c == -1:
+            continue
+        color_idx = label_to_idx[c]
+        fig_routes.add_trace(
+            go.Scattergeo(
+                lon=lons,
+                lat=lats,
+                mode='lines',
+                opacity=0.3,
+                line=dict(width=1, color=cluster_colors[color_idx % len(cluster_colors)]),
+                showlegend=False
+            )
+        )
+    for i, c in enumerate(unique_labels):
+        mean_lon, mean_lat = cluster_mean_routes[i]
+        if len(mean_lon) == 0:
+            continue
+        end_reg = endpoint_region_label(c, best_labels, filtered_storms)
+        name = f"Cluster {i+1}" + (f" (towards {end_reg})" if end_reg else "")
+        fig_routes.add_trace(
+            go.Scattergeo(
+                lon=mean_lon,
+                lat=mean_lat,
+                mode='lines',
+                line=dict(width=4, color=cluster_colors[i % len(cluster_colors)]),
+                name=name
+            )
+        )
+        fig_routes.add_trace(
+            go.Scattergeo(
+                lon=[mean_lon[0]],
+                lat=[mean_lat[0]],
+                mode='markers',
+                marker=dict(size=10, color='green', symbol='triangle-up'),
+                name=f"Cluster {i+1} Start"
+            )
+        )
+        fig_routes.add_trace(
+            go.Scattergeo(
+                lon=[mean_lon[-1]],
+                lat=[mean_lat[-1]],
+                mode='markers',
+                marker=dict(size=10, color='red', symbol='x'),
+                name=f"Cluster {i+1} End"
+            )
+        )
+    enso_phase_text = {'all': 'All Years', 'El Nino': 'El Niño', 'La Nina': 'La Niña', 'Neutral': 'Neutral Years'}
     fig_routes.update_layout(
-        title="Typhoon Routes by Cluster",
+        title=f"West Pacific Typhoon Routes ({start_year}-{end_year}, {season.capitalize()}, {enso_phase_text.get(enso_value, 'All Years')})",
         geo=dict(scope='asia', projection_type='mercator', showland=True, landcolor='lightgray')
     )
 
     # Cluster Statistics Plot
-    cluster_stats = []
-    for cluster in set(best_labels) - {-1}:
-        mask = best_labels == cluster
-        winds = [all_storms_data[i][2].max() for i in range(len(all_storms_data)) if mask[i]]
-        pressures = [all_storms_data[i][3].min() for i in range(len(all_storms_data)) if mask[i]]
-        cluster_stats.append({
-            'Cluster': cluster,
-            'Count': np.sum(mask),
-            'Mean Wind': np.mean(winds),
-            'Mean Pressure': np.mean(pressures)
-        })
-    stats_df = pd.DataFrame(cluster_stats)
-    fig_stats = go.Figure()
-    fig_stats.add_trace(go.Bar(x=stats_df['Cluster'], y=stats_df['Count'], name='Storm Count'))
-    fig_stats.add_trace(go.Bar(x=stats_df['Cluster'], y=stats_df['Mean Wind'], name='Mean Max Wind Speed'))
-    fig_stats.add_trace(go.Bar(x=stats_df['Cluster'], y=stats_df['Mean Pressure'], name='Mean Min Pressure'))
-    fig_stats.update_layout(barmode='group', title="Cluster Statistics")
+    fig_stats = make_subplots(rows=2, cols=1, shared_xaxes=True, subplot_titles=("Average Wind Speed", "Average Pressure"))
+    for i, c in enumerate(unique_labels):
+        if len(cluster_mean_vmax[i]) > 0:
+            end_reg = endpoint_region_label(c, best_labels, filtered_storms)
+            name = f"Cluster {i+1}" + (f" ({end_reg})" if end_reg else "")
+            fig_stats.add_trace(
+                go.Scatter(y=cluster_mean_vmax[i], mode='lines', line=dict(width=2, color=cluster_colors[i % len(cluster_colors)]), name=name),
+                row=1, col=1
+            )
+            fig_stats.add_trace(
+                go.Scatter(y=cluster_mean_mslp[i], mode='lines', line=dict(width=2, color=cluster_colors[i % len(cluster_colors)]), name=name),
+                row=2, col=1
+            )
+    fig_stats.update_layout(
+        title="Cluster Average Wind & Pressure Profiles",
+        xaxis_title="Route Normalized Index",
+        yaxis_title="Wind Speed (knots)",
+        xaxis2_title="Route Normalized Index",
+        yaxis2_title="Pressure (hPa)",
+        showlegend=True,
+        legend_tracegroupgap=300
+    )
 
     # Cluster Information
-    cluster_info = f"Date Range: {start_year}-{start_month} to {end_year}-{end_month}\nENSO Phase: {enso_value}\nSeason: {season}\n\n"
-    cluster_info += f"Selected EPS: {best_eps}\nNumber of Clusters: {best_n_clusters}\nNoise Points: {best_n_noise} ({(best_n_noise / len(best_labels))*100:.1f}%)\n"
-    for stat in cluster_stats:
-        cluster_info += f"Cluster {stat['Cluster']}: {stat['Count']} storms, Mean Max Wind: {stat['Mean Wind']:.1f} kt, Mean Min Pressure: {stat['Mean Pressure']:.1f} hPa\n"
+    cluster_info_lines = [f"Selected DBSCAN eps: {best_eps:.2f}", f"Number of noise points: {best_n_noise}"]
+    for i, c in enumerate(unique_labels):
+        indices = np.where(best_labels == c)[0]
+        count = len(indices)
+        if count == 0:
+            continue
+        avg_durations = {r: (cluster_region_durations[i][r] / count) for r in cluster_region_durations[i]}
+        end_reg = endpoint_region_label(c, best_labels, filtered_storms)
+        name = f"Cluster {i+1}" + (f" (towards {end_reg})" if end_reg else "")
+        cluster_info_lines.append(f"\n{name}")
+        if avg_durations:
+            for reg, hrs in avg_durations.items():
+                cluster_info_lines.append(f"{reg}: {hrs:.2f} hours")
+        else:
+            cluster_info_lines.append("No significant region durations.")
+        if end_reg in ["Taiwan Land", "Taiwan Sea"] and len(cluster_mean_vmax[i]) > 0:
+            final_wind = cluster_mean_vmax[i][-1]
+            if final_wind >= 34:
+                cluster_info_lines.append(
+                    "CWA would issue a land warning ~18 hours before arrival." if end_reg == "Taiwan Land"
+                    else "CWA would issue a sea warning ~24 hours before arrival."
+                )
+    if len(noise_indices) > 0:
+        cluster_info_lines.append(f"\nNoise Cluster\nNumber of storms classified as noise: {len(noise_indices)}")
 
-    return fig_tsne, fig_routes, fig_stats, cluster_info
+    cluster_info_text = "\n".join(cluster_info_lines)
+    return fig_tsne, fig_routes, fig_stats, cluster_info_text
 
 # Gradio Interface
 with gr.Blocks(title="Typhoon Analysis Dashboard") as demo:
@@ -587,7 +826,7 @@ with gr.Blocks(title="Typhoon Analysis Dashboard") as demo:
         - **Pressure Analysis**: Analyze pressure vs ONI relationships
         - **Longitude Analysis**: Study typhoon generation longitude vs ONI
         - **Path Animation**: Watch animated typhoon paths with a sidebar
-        - **TSNE Cluster**: Perform t-SNE clustering on typhoon routes
+        - **TSNE Cluster**: Perform t-SNE clustering on typhoon routes with mean routes and region analysis
         
         Select a tab above to begin your analysis.
         """)
@@ -783,7 +1022,7 @@ with gr.Blocks(title="Typhoon Analysis Dashboard") as demo:
             tsne_season = gr.Dropdown(label="Season", choices=['all', 'summer', 'winter'], value='all')
         tsne_analyze_btn = gr.Button("Analyze")
         tsne_plot = gr.Plot(label="t-SNE Clusters")
-        routes_plot = gr.Plot(label="Typhoon Routes")
+        routes_plot = gr.Plot(label="Typhoon Routes with Mean Routes")
         stats_plot = gr.Plot(label="Cluster Statistics")
         cluster_info = gr.Textbox(label="Cluster Information", lines=10)
         
@@ -793,20 +1032,4 @@ with gr.Blocks(title="Typhoon Analysis Dashboard") as demo:
             outputs=[tsne_plot, routes_plot, stats_plot, cluster_info]
         )
 
-    # Custom CSS for better visibility
-    gr.HTML("""
-    <style>
-    #tracks_plot, #path_video {
-        height: 700px !important;
-        width: 100%;
-    }
-    .plot-container {
-        min-height: 600px;
-    }
-    .gr-plotly {
-        width: 100% !important;
-    }
-    </style>
-    """)
-
 demo.launch(share=True)