Update app.py

app.py

@@ -3,54 +3,129 @@ import pandas as pd
 import numpy as np
 from datetime import datetime
 import re
+from playwright.sync_api import sync_playwright
+import time
+import os
+import subprocess
+import sys
 
-def parse_weather_data(text):
-    """Parse the weather data text into a structured format"""
-    lines = [line.strip() for line in text.split('\n') if line.strip()]
-    data = []
+# Install Playwright browsers on startup
+def install_playwright_browsers():
+    try:
+        if not os.path.exists('/home/user/.cache/ms-playwright'):
+            print("Installing Playwright browsers...")
+            subprocess.run(
+                [sys.executable, "-m", "playwright", "install", "chromium"],
+                check=True,
+                capture_output=True,
+                text=True
+            )
+            print("Playwright browsers installed successfully")
+    except Exception as e:
+        print(f"Error installing browsers: {e}")
+
+# Install browsers when the module loads
+install_playwright_browsers()
+
+def scrape_weather_data(site_id, hours=720):
+    """Scrape weather data from weather.gov timeseries"""
+    url = f"https://www.weather.gov/wrh/timeseries?site={site_id}&hours={hours}&units=english&chart=on&headers=on&obs=tabular&hourly=false&pview=full&font=12&plot="
     
-    for line in lines:
-        if not ('am' in line or 'pm' in line):
-            continue
-            
-        parts = line.split('\t')
-        if len(parts) < 8:
-            continue
-            
-        try:
-            # Parse wind speed and gusts (format: "12G30")
-            wind_match = re.search(r'(\d+)G(\d+)', parts[6]) if len(parts) > 6 else None
-            wind_speed = int(wind_match.group(1)) if wind_match else np.nan
-            wind_gust = int(wind_match.group(2)) if wind_match else np.nan
+    with sync_playwright() as p:
+        # Launch browser with minimal settings
+        browser = p.chromium.launch(
+            headless=True,
+            args=['--no-sandbox', '--disable-dev-shm-usage']
+        )
+        
+        # Create context with desktop user agent
+        context = browser.new_context(
+            user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        )
+        
+        # Create new page and navigate
+        page = context.new_page()
+        page.goto(url)
+        
+        # Wait for content to load
+        time.sleep(5)
+        
+        # Get all text content
+        content = page.evaluate('''() => {
+            // Function to get all text content
+            const getTextContent = () => {
+                const rows = [];
+                const tables = document.getElementsByTagName('table');
+                for (const table of tables) {
+                    if (table.textContent.includes('Date/Time')) {
+                        const headerRow = Array.from(table.querySelectorAll('th'))
+                            .map(th => th.textContent.trim());
+                        
+                        const dataRows = Array.from(table.querySelectorAll('tbody tr'))
+                            .map(row => Array.from(row.querySelectorAll('td'))
+                                .map(td => td.textContent.trim()));
+                        
+                        return {headers: headerRow, rows: dataRows};
+                    }
+                }
+                return null;
+            };
             
-            row = {
-                'datetime': parts[0].strip(),
-                'temperature': float(parts[1]),
-                'dew_point': float(parts[2]),
-                'humidity': float(parts[3]),
-                'wind_chill': float(parts[4]) if parts[4].strip() != '' else np.nan,
-                'wind_speed': wind_speed,
-                'wind_gust': wind_gust,
-                'snow_depth': float(parts[7]) if parts[7].strip() != '' else np.nan
-            }
-            data.append(row)
-        except:
-            continue
+            return getTextContent();
+        }''')
+        
+        browser.close()
+        return content
+
+def parse_weather_data(data):
+    """Parse the weather data into a pandas DataFrame"""
+    if not data or 'rows' not in data:
+        raise ValueError("No valid weather data found")
+        
+    # Convert to DataFrame
+    df = pd.DataFrame(data['rows'])
+    
+    # Assign column names (first 8 columns we care about)
+    columns = ['datetime', 'temp', 'dew_point', 'humidity', 'wind_chill', 'wind_dir', 'wind_speed', 'snow_depth']
+    df = df.iloc[:, :8]
+    df.columns = columns
     
-    return pd.DataFrame(data)
+    # Convert numeric columns
+    df['temp'] = pd.to_numeric(df['temp'], errors='coerce')
+    df['dew_point'] = pd.to_numeric(df['dew_point'], errors='coerce')
+    df['humidity'] = pd.to_numeric(df['humidity'], errors='coerce')
+    df['wind_chill'] = pd.to_numeric(df['wind_chill'], errors='coerce')
+    df['snow_depth'] = pd.to_numeric(df['snow_depth'], errors='coerce')
+    
+    # Parse wind speed and gusts
+    def parse_wind(x):
+        if pd.isna(x): return np.nan, np.nan
+        match = re.search(r'(\d+)G(\d+)', str(x))
+        if match:
+            return float(match.group(1)), float(match.group(2))
+        return np.nan, np.nan
+    
+    wind_data = df['wind_speed'].apply(parse_wind)
+    df['wind_speed'] = wind_data.apply(lambda x: x[0])
+    df['wind_gust'] = wind_data.apply(lambda x: x[1])
+    
+    return df
 
-def analyze_weather_data(data_str):
+def analyze_weather_data(site_id, hours):
     """Analyze weather data and create visualizations"""
     try:
+        # Scrape data
+        raw_data = scrape_weather_data(site_id, hours)
+        if not raw_data:
+            return "Error: Could not retrieve weather data.", None, None
+            
         # Parse data
-        df = parse_weather_data(data_str)
-        if len(df) == 0:
-            return "Error: No valid data found in input."
+        df = parse_weather_data(raw_data)
         
         # Calculate statistics
         stats = {
-            'Temperature Range': f"{df['temperature'].min():.1f}°F to {df['temperature'].max():.1f}°F",
-            'Average Temperature': f"{df['temperature'].mean():.1f}°F",
+            'Temperature Range': f"{df['temp'].min():.1f}°F to {df['temp'].max():.1f}°F",
+            'Average Temperature': f"{df['temp'].mean():.1f}°F",
             'Max Wind Speed': f"{df['wind_speed'].max():.1f} mph",
             'Max Wind Gust': f"{df['wind_gust'].max():.1f} mph",
             'Average Humidity': f"{df['humidity'].mean():.1f}%",
@@ -59,13 +134,14 @@ def analyze_weather_data(data_str):
         
         # Create HTML output
         html_output = "<div style='font-size: 16px; line-height: 1.5;'>"
+        html_output += f"<p><strong>Weather Station:</strong> {site_id}</p>"
         for key, value in stats.items():
             html_output += f"<p><strong>{key}:</strong> {value}</p>"
         html_output += "</div>"
         
         # Create temperature plot
         temp_fig = gr.Plot()
-        df.plot(x='datetime', y=['temperature', 'wind_chill'],
+        df.plot(x='datetime', y=['temp', 'wind_chill'],
                 title='Temperature and Wind Chill Over Time',
                 figsize=(12, 6))
         temp_fig.pyplot()
@@ -83,18 +159,30 @@ def analyze_weather_data(data_str):
         return f"Error analyzing data: {str(e)}", None, None
 
 # Create Gradio interface
-with gr.Blocks(title="Weather Data Analyzer") as demo:
-    gr.Markdown("# Weather Data Analyzer")
-    gr.Markdown("Paste weather data from weather.gov/wrh/timeseries in the text box below.")
+with gr.Blocks(title="Weather Station Data Analyzer") as demo:
+    gr.Markdown("# Weather Station Data Analyzer")
+    gr.Markdown("""
+    Enter a weather station ID and number of hours to analyze.
+    Example station IDs:
+    - YCTIM (Yellowstone Club - Timber)
+    - KBZN (Bozeman Airport)
+    - KSLC (Salt Lake City)
+    """)
     
     with gr.Row():
-        input_text = gr.Textbox(
-            label="Weather Data",
-            placeholder="Paste weather data here...",
-            lines=10
+        site_id = gr.Textbox(
+            label="Weather Station ID",
+            value="YCTIM",
+            placeholder="Enter station ID (e.g., YCTIM)"
+        )
+        hours = gr.Number(
+            label="Hours of Data",
+            value=720,
+            minimum=1,
+            maximum=1440
         )
     
-    analyze_btn = gr.Button("Analyze Weather Data")
+    analyze_btn = gr.Button("Fetch and Analyze Weather Data")
     
     with gr.Row():
         stats_output = gr.HTML(label="Statistics")
@@ -105,7 +193,7 @@ with gr.Blocks(title="Weather Data Analyzer") as demo:
     
     analyze_btn.click(
         fn=analyze_weather_data,
-        inputs=[input_text],
+        inputs=[site_id, hours],
         outputs=[stats_output, temp_plot, wind_plot]
     )