Update app.py

app.py

@@ -9,7 +9,6 @@ import subprocess
 import sys
 import matplotlib.pyplot as plt
 from matplotlib.gridspec import GridSpec
-import matplotlib.dates as mdates
 from windrose import WindroseAxes
 from datetime import datetime
 
@@ -37,13 +36,18 @@ def scrape_weather_data(site_id, hours=720):
     
     try:
         with sync_playwright() as p:
+            # Launch browser with minimal settings
             browser = p.chromium.launch(
                 headless=True,
                 args=['--no-sandbox', '--disable-dev-shm-usage']
             )
+            
+            # Create context with desktop user agent
             context = browser.new_context(
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
             )
+            
+            # Create new page and navigate
             page = context.new_page()
             response = page.goto(url)
             print(f"Response status: {response.status}")
@@ -55,21 +59,28 @@ def scrape_weather_data(site_id, hours=720):
             # Get all text content
             print("Extracting data...")
             content = page.evaluate('''() => {
-                const rows = [];
-                const tables = document.getElementsByTagName('table');
-                for (const table of tables) {
-                    if (table.textContent.includes('Date/Time')) {
-                        const headerRow = Array.from(table.querySelectorAll('th'))
-                            .map(th => th.textContent.trim());
-                        const dataRows = Array.from(table.querySelectorAll('tbody tr'))
-                            .map(row => Array.from(row.querySelectorAll('td'))
-                                .map(td => td.textContent.trim()));
-                        return {headers: headerRow, rows: dataRows};
+                const getTextContent = () => {
+                    const rows = [];
+                    const tables = document.getElementsByTagName('table');
+                    for (const table of tables) {
+                        if (table.textContent.includes('Date/Time')) {
+                            const headerRow = Array.from(table.querySelectorAll('th'))
+                                .map(th => th.textContent.trim());
+                            
+                            const dataRows = Array.from(table.querySelectorAll('tbody tr'))
+                                .map(row => Array.from(row.querySelectorAll('td'))
+                                    .map(td => td.textContent.trim()));
+                            
+                            return {headers: headerRow, rows: dataRows};
+                        }
                     }
-                }
-                return null;
+                    return null;
+                };
+                
+                return getTextContent();
             }''')
             
+            print(f"Found {len(content['rows'] if content else [])} rows of data")
             browser.close()
             return content
             
@@ -77,6 +88,16 @@ def scrape_weather_data(site_id, hours=720):
         print(f"Error scraping data: {str(e)}")
         raise e
 
+def parse_date(date_str):
+    """Parse date string to datetime"""
+    try:
+        # Handle format like "Feb 10, 10:00 am"
+        # Add current year to the date string
+        current_year = datetime.now().year
+        return pd.to_datetime(f"{date_str}, {current_year}", format="%b %d, %I:%M %p, %Y")
+    except:
+        return pd.NaT
+
 def parse_weather_data(data):
     """Parse the weather data into a pandas DataFrame"""
     if not data or 'rows' not in data:
@@ -123,137 +144,80 @@ def parse_weather_data(data):
     df['wind_dir_deg'] = df['wind_dir'].apply(parse_direction)
     
     # Convert datetime
-    current_year = datetime.now().year
-    df['datetime'] = pd.to_datetime(df['datetime'].apply(lambda x: f"{x}, {current_year}"), 
-                                  format="%b %d, %I:%M %p, %Y")
+    df['datetime'] = df['datetime'].apply(parse_date)
     df['date'] = df['datetime'].dt.date
     
     return df
 
-def create_wind_rose(df, ax=None):
+def create_wind_rose(df, ax):
     """Create a wind rose plot"""
-    if ax is None:
-        fig = plt.figure(figsize=(8, 8))
-        ax = WindroseAxes.from_ax(fig=fig)
-    
-    ws = df['wind_speed'].dropna().values
-    wd = df['wind_dir_deg'].dropna().values
-    
-    if len(ws) > 0 and len(wd) > 0:
-        # Define wind speed bins
-        bins = np.array([0, 5, 10, 15, 20, 25, 30, 35])
-        # Create color map
-        colors = plt.cm.viridis(np.linspace(0, 1, len(bins)-1))
-        
-        # Create the wind rose
-        ax.bar(wd, ws, nsector=36, normed=True, opening=0.8,
-              bins=bins, colors=colors)
-        ax.set_legend(title='Wind Speed (mph)', bbox_to_anchor=(1.2, 0.5))
-        return True
-    return False
-
-def create_daily_wind_roses(df):
-    """Create wind roses for each day"""
-    daily_groups = df.groupby(df['datetime'].dt.date)
-    roses = []
-    
-    for date, group in daily_groups:
-        if len(group) > 0:  # Only create rose if we have data
-            fig = plt.figure(figsize=(8, 8))
-            ax = WindroseAxes.from_ax(fig=fig)
-            if create_wind_rose(group, ax):
-                ax.set_title(f'Wind Rose - {date.strftime("%Y-%m-%d")}')
-                plt.tight_layout()
-                roses.append(fig)
-            plt.close(fig)
-    
-    return roses
+    if not isinstance(ax, WindroseAxes):
+        ax = WindroseAxes.from_ax(ax=ax)
+    ax.bar(df['wind_dir_deg'].dropna(), df['wind_speed'].dropna(), 
+           bins=np.arange(0, 40, 5), normed=True, opening=0.8, edgecolor='white')
+    ax.set_legend(title='Wind Speed (mph)')
+    ax.set_title('Wind Rose')
 
 def create_plots(df):
     """Create all weather plots"""
-    # Create main figure with subplots
-    fig = plt.figure(figsize=(20, 25))
-    gs = GridSpec(4, 2, figure=fig, height_ratios=[1, 1, 1, 1])
+    # Create figure with subplots
+    fig = plt.figure(figsize=(20, 15))
+    gs = GridSpec(3, 2, figure=fig)
     
     # Temperature plot
     ax1 = fig.add_subplot(gs[0, :])
     ax1.plot(df['datetime'], df['temp'], label='Temperature', color='red')
     ax1.plot(df['datetime'], df['wind_chill'], label='Wind Chill', color='blue')
-    ax1.set_title('Temperature and Wind Chill Over Time', pad=20)
+    ax1.set_title('Temperature and Wind Chill Over Time')
     ax1.set_xlabel('Date')
     ax1.set_ylabel('Temperature (°F)')
     ax1.legend()
     ax1.grid(True)
-    ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d %H:%M'))
-    plt.setp(ax1.xaxis.get_majorticklabels(), rotation=45, ha='right')
+    plt.setp(ax1.xaxis.get_majorticklabels(), rotation=45)
     
     # Wind speed plot
     ax2 = fig.add_subplot(gs[1, :])
     ax2.plot(df['datetime'], df['wind_speed'], label='Wind Speed', color='blue')
     ax2.plot(df['datetime'], df['wind_gust'], label='Wind Gust', color='orange')
-    ax2.set_title('Wind Speed and Gusts Over Time', pad=20)
+    ax2.set_title('Wind Speed and Gusts Over Time')
     ax2.set_xlabel('Date')
     ax2.set_ylabel('Wind Speed (mph)')
     ax2.legend()
     ax2.grid(True)
-    ax2.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d %H:%M'))
-    plt.setp(ax2.xaxis.get_majorticklabels(), rotation=45, ha='right')
+    plt.setp(ax2.xaxis.get_majorticklabels(), rotation=45)
     
     # Snow depth plot
     ax3 = fig.add_subplot(gs[2, 0])
     ax3.plot(df['datetime'], df['snow_depth'], color='blue', label='Snow Depth')
-    ax3.set_title('Snow Depth Over Time', pad=20)
+    ax3.set_title('Snow Depth Over Time')
     ax3.set_xlabel('Date')
     ax3.set_ylabel('Snow Depth (inches)')
     ax3.grid(True)
-    ax3.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d %H:%M'))
-    plt.setp(ax3.xaxis.get_majorticklabels(), rotation=45, ha='right')
+    plt.setp(ax3.xaxis.get_majorticklabels(), rotation=45)
     
     # Daily new snow bar plot
     ax4 = fig.add_subplot(gs[2, 1])
     daily_snow = df.groupby('date')['snowfall_24hr'].max()
-    bars = ax4.bar(range(len(daily_snow)), daily_snow.values, color='blue', width=0.8)
-    
-    # Customize x-axis
-    ax4.set_xticks(range(len(daily_snow)))
-    ax4.set_xticklabels([d.strftime('%Y-%m-%d') for d in daily_snow.index], 
-                        rotation=45, ha='right')
-    
-    # Add value labels on top of bars
-    for bar in bars:
-        height = bar.get_height()
-        if height > 0:  # Only label bars with snow
-            ax4.text(bar.get_x() + bar.get_width()/2., height,
-                    f'{height:.1f}"',
-                    ha='center', va='bottom')
-    
-    ax4.set_title('Daily New Snow', pad=20)
+    ax4.bar(daily_snow.index, daily_snow.values, color='blue')
+    ax4.set_title('Daily New Snow')
     ax4.set_xlabel('Date')
     ax4.set_ylabel('New Snow (inches)')
-    ax4.grid(True, axis='y')
-    
-    # Wind roses
-    ax5 = fig.add_subplot(gs[3, 0], projection='windrose')
-    create_wind_rose(df, ax5)
-    ax5.set_title('Overall Wind Rose')
-    
-    # Latest day's wind rose
-    latest_date = df['date'].iloc[0]
-    latest_data = df[df['date'] == latest_date]
-    ax6 = fig.add_subplot(gs[3, 1], projection='windrose')
-    create_wind_rose(latest_data, ax6)
-    ax6.set_title(f'Latest Day Wind Rose\n{latest_date}')
+    plt.setp(ax4.xaxis.get_majorticklabels(), rotation=45)
     
     plt.tight_layout()
     
-    # Create individual daily wind roses
-    daily_roses = create_daily_wind_roses(df)
+    # Create separate wind rose figure
+    fig_rose = plt.figure(figsize=(10, 10))
+    ax_rose = WindroseAxes.from_ax(fig=fig_rose)
+    create_wind_rose(df, ax_rose)
+    plt.tight_layout()
     
-    return fig, daily_roses
+    return fig, fig_rose
 
 def analyze_weather_data(site_id, hours):
     """Analyze weather data and create visualizations"""
     try:
+        # Scrape and parse data
         print(f"Scraping data for {site_id}...")
         raw_data = scrape_weather_data(site_id, hours)
         if not raw_data:
@@ -284,9 +248,9 @@ def analyze_weather_data(site_id, hours):
         
         # Create plots
         print("Creating plots...")
-        main_plots, daily_roses = create_plots(df)
+        main_plots, wind_rose = create_plots(df)
         
-        return html_output, main_plots, daily_roses
+        return html_output, main_plots, wind_rose
         
     except Exception as e:
         print(f"Error in analysis: {str(e)}")
@@ -323,13 +287,12 @@ with gr.Blocks(title="Weather Station Data Analyzer") as demo:
     
     with gr.Row():
         weather_plots = gr.Plot(label="Weather Plots")
-    
-    gallery = gr.Gallery(label="Daily Wind Roses", columns=3, rows=2, height=400)
+        wind_rose = gr.Plot(label="Wind Rose")
     
     analyze_btn.click(
         fn=analyze_weather_data,
         inputs=[site_id, hours],
-        outputs=[stats_output, weather_plots, gallery]
+        outputs=[stats_output, weather_plots, wind_rose]
     )
 
 if __name__ == "__main__":