Update app.py

app.py

@@ -1,365 +1,66 @@
-# app.py
-import gradio as gr
+# scrape_snow_depth.py
 import requests
-import pandas as pd
-import numpy as np
-from datetime import datetime, timedelta
-import matplotlib.pyplot as plt
-from matplotlib.gridspec import GridSpec
-import json
 from bs4 import BeautifulSoup
-
-def get_raw_data(station_id):
-    """
-    Get raw data from the NWS API
-    """
-    headers = {
-        'User-Agent': '(Weather Data Viewer, contact@yourdomain.com)',
-        'Accept': 'application/json'
-    }
-    
-    # Calculate correct date range for last 3 days
-    end_time = datetime.utcnow()
-    start_time = end_time - timedelta(hours=72)  # Last 3 days
-    
-    params = {
-        'start': start_time.isoformat() + 'Z',
-        'end': end_time.isoformat() + 'Z'
-    }
-    
-    url = f"https://api.weather.gov/stations/{station_id}/observations"
-    
-    try:
-        print("\nFetching observations...")
-        print(f"URL: {url}")
-        print(f"Time range: {start_time} to {end_time}")
-        response = requests.get(url, headers=headers, params=params)
-        print(f"Response status: {response.status_code}")
-        
-        if response.status_code != 200:
-            print(f"Response content: {response.text}")
-            response.raise_for_status()
-        
-        data = response.json()
-        
-        if 'features' in data:
-            print(f"\nNumber of observations: {len(data['features'])}")
-            if len(data['features']) > 0:
-                print("\nFirst observation properties:")
-                print(json.dumps(data['features'][0]['properties'], indent=2))
-                
-                print("\nAll available property keys:")
-                keys = set()
-                for feature in data['features']:
-                    keys.update(feature['properties'].keys())
-                print(sorted(list(keys)))
-        
-        return data
-    except Exception as e:
-        print(f"Error fetching data: {e}")
-        print("Full error details:")
-        import traceback
-        traceback.print_exc()
-        return None
+import pandas as pd
 
 def scrape_snow_depth():
     """
-    Scrape snow depth data from the HTML timeseries page.
-    Note: The structure of the HTML page may change over time.
+    Scrapes the snow depth data from the weather.gov timeseries page for station YCTIM.
+    It extracts the 'Time' and 'Snow Depth' columns from the table.
     """
     url = "https://www.weather.gov/wrh/timeseries?site=YCTIM&hours=720&units=english&chart=on&headers=on&obs=tabular&hourly=false&pview=standard&font=12&plot="
-    try:
-        response = requests.get(url)
-        if response.status_code != 200:
-            print(f"Failed to fetch HTML page: {response.status_code}")
-            return pd.DataFrame()
-        soup = BeautifulSoup(response.text, 'html.parser')
-        table = soup.find('table')
-        if table is None:
-            print("No table found on the page.")
-            return pd.DataFrame()
-        
-        # Extract headers from the table
-        header_row = table.find('tr')
-        headers = [th.get_text().strip() for th in header_row.find_all('th')]
-        
-        # Look for the columns "Time" and "Snow Depth"
-        if "Time" not in headers or "Snow Depth" not in headers:
-            print("Required columns 'Time' or 'Snow Depth' not found in table headers.")
-            return pd.DataFrame()
-        
-        time_idx = headers.index("Time")
-        snow_idx = headers.index("Snow Depth")
-        
-        data = []
-        rows = table.find_all('tr')[1:]  # Skip header row
-        for row in rows:
-            cells = row.find_all('td')
-            if len(cells) < max(time_idx, snow_idx) + 1:
-                continue
-            time_str = cells[time_idx].get_text().strip()
-            snow_str = cells[snow_idx].get_text().strip()
-            data.append((time_str, snow_str))
-        
-        df = pd.DataFrame(data, columns=["Time", "snowDepth"])
-        # Convert "Time" column to datetime. Adjust the format if necessary.
-        try:
-            df["timestamp"] = pd.to_datetime(df["Time"])
-        except Exception as e:
-            print("Error parsing timestamp from scraped data:", e)
-            df["timestamp"] = pd.NaT
-        df["snowDepth"] = pd.to_numeric(df["snowDepth"], errors='coerce')
-        print("Scraped snow depth data:")
-        print(df.head())
-        return df[["timestamp", "snowDepth"]]
-    except Exception as e:
-        print(f"Error scraping snow depth: {e}")
-        return pd.DataFrame()
-
-def parse_raw_data(data):
-    """
-    Parse the raw JSON data into a DataFrame
-    """
-    if not data or 'features' not in data:
-        return None
-
-    records = []
-    for feature in data['features']:
-        props = feature['properties']
-        
-        # Extract all properties starting with 'snow'
-        snow_fields = {k: v for k, v in props.items() if 'snow' in k.lower()}
-        if snow_fields:
-            print("\nFound snow-related fields:")
-            for k, v in snow_fields.items():
-                print(f"{k}: {v}")
-        
-        record = {
-            'timestamp': props['timestamp'],
-            'temperature': props.get('temperature', {}).get('value'),
-            'wind_speed': props.get('windSpeed', {}).get('value'),
-            'wind_direction': props.get('windDirection', {}).get('value')
-        }
-        
-        # Add all snow-related fields to the record
-        for k, v in snow_fields.items():
-            if isinstance(v, dict) and 'value' in v:
-                record[k] = v['value']
-            else:
-                record[k] = v
-        
-        records.append(record)
-
-    df = pd.DataFrame(records)
     
-    print("\nDataFrame columns:")
-    print(df.columns.tolist())
-    print("\nSample of raw data:")
-    print(df.head())
-    
-    return df
-
-def process_weather_data(df):
-    """
-    Process the weather DataFrame
-    """
-    if df is None or df.empty:
+    # Get the webpage content
+    response = requests.get(url)
+    if response.status_code != 200:
+        print(f"Error: Could not fetch the page. Status code: {response.status_code}")
         return None
-
-    # Convert timestamp
-    df['timestamp'] = pd.to_datetime(df['timestamp'])
-    df['date'] = df['timestamp'].dt.date
-
-    # Convert temperature from C to F if not null
-    if df['temperature'].notna().all():
-        df['temperature'] = (df['temperature'] * 9/5) + 32
-
-    # Convert wind speed from km/h to mph if not null (original unit km/h)
-    if df['wind_speed'].notna().all():
-        df['wind_speed'] = df['wind_speed'] * 0.621371  # km/h to mph
-
-    return df
-
-def create_wind_rose(ax, data, title):
-    """
-    Create a wind rose subplot
-    """
-    if data.empty or data['wind_direction'].isna().all() or data['wind_speed'].isna().all():
-        ax.text(0.5, 0.5, 'No wind data available', 
-                horizontalalignment='center',
-                verticalalignment='center',
-                transform=ax.transAxes)
-        ax.set_title(title)
-        return
-
-    plot_data = data.copy()
-    
-    direction_bins = np.arange(0, 361, 45)
-    directions = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW']
-    
-    mask = plot_data['wind_direction'].notna() & plot_data['wind_speed'].notna()
-    plot_data = plot_data[mask]
     
-    if plot_data.empty:
-        ax.text(0.5, 0.5, 'No valid wind data', 
-                horizontalalignment='center',
-                verticalalignment='center',
-                transform=ax.transAxes)
-        ax.set_title(title)
-        return
+    # Parse the HTML content
+    soup = BeautifulSoup(response.text, "html.parser")
     
-    plot_data.loc[:, 'direction_bin'] = pd.cut(plot_data['wind_direction'], 
-                                             bins=direction_bins, 
-                                             labels=directions,
-                                             include_lowest=True)
-    
-    wind_stats = plot_data.groupby('direction_bin', observed=True)['wind_speed'].mean()
-    
-    all_directions = pd.Series(0.0, index=directions)
-    wind_stats = wind_stats.combine_first(all_directions)
-    
-    angles = np.linspace(0, 2*np.pi, len(directions), endpoint=False)
-    values = [wind_stats[d] for d in directions]
-    
-    if any(v > 0 for v in values):
-        bars = ax.bar(angles, values, width=0.5, alpha=0.6)
-        ax.set_xticks(angles)
-        ax.set_xticklabels(directions)
-    else:
-        ax.text(0.5, 0.5, 'No significant wind', 
-                horizontalalignment='center',
-                verticalalignment='center',
-                transform=ax.transAxes)
-    
-    ax.set_title(title)
+    # Find the first table on the page
+    table = soup.find("table")
+    if table is None:
+        print("No table found on the page.")
+        return None
 
-def create_visualizations(df):
-    """
-    Create static visualizations using matplotlib
-    """
-    fig = plt.figure(figsize=(20, 24))
-    gs = GridSpec(5, 2, figure=fig)
-    
-    ax1 = fig.add_subplot(gs[0, :])
-    ax2 = fig.add_subplot(gs[1, :])
-    ax3 = fig.add_subplot(gs[2, :])
-    
-    if not df['temperature'].isna().all():
-        ax1.plot(df['timestamp'], df['temperature'], linewidth=2)
-    ax1.set_title('Temperature Over Time')
-    ax1.set_ylabel('Temperature (°F)')
-    ax1.set_xlabel('')
-    ax1.grid(True)
-    
-    if not df['wind_speed'].isna().all():
-        ax2.plot(df['timestamp'], df['wind_speed'], linewidth=2)
-    ax2.set_title('Wind Speed Over Time')
-    ax2.set_ylabel('Wind Speed (mph)')
-    ax2.set_xlabel('')
-    ax2.grid(True)
-    
-    # Plot snow depth if available
-    if 'snowDepth' in df.columns and not df['snowDepth'].isna().all():
-        ax3.plot(df['timestamp'], df['snowDepth'], linewidth=2)
-        ax3.set_ylim(0, 80)  # Fixed y-axis limit to 80 inches
-    else:
-        ax3.text(0.5, 0.5, 'No snow depth data available', 
-                 horizontalalignment='center',
-                 verticalalignment='center',
-                 transform=ax3.transAxes)
-    ax3.set_title('Snow Depth')
-    ax3.set_ylabel('Snow Depth (inches)')
-    ax3.set_xlabel('')
-    ax3.grid(True)
-    
-    for ax in [ax1, ax2, ax3]:
-        ax.tick_params(axis='x', rotation=45)
-        ax.xaxis.set_major_formatter(plt.matplotlib.dates.DateFormatter('%Y-%m-%d %H:%M'))
-    
-    dates = sorted(df['date'].unique())
-    wind_axes = []
-    for i, date in enumerate(dates):
-        if i < 2:
-            ax = fig.add_subplot(gs[4, i], projection='polar')
-            day_data = df[df['date'] == date].copy()
-            create_wind_rose(ax, day_data, pd.to_datetime(date).strftime('%Y-%m-%d'))
-            wind_axes.append(ax)
+    # Extract the header row to identify the column indices for 'Time' and 'Snow Depth'
+    header_row = table.find("tr")
+    headers = [th.get_text(strip=True) for th in header_row.find_all("th")]
     
-    plt.tight_layout()
-    
-    return fig
-
-def get_weather_data(station_id, hours):
-    """
-    Main function to get and process weather data
-    """
-    try:
-        # Get raw data from API
-        raw_data = get_raw_data(station_id)
-        if raw_data is None:
-            return None, "Failed to fetch data"
-
-        # Parse raw data
-        df = parse_raw_data(raw_data)
-        if df is None:
-            return None, "Failed to parse data"
-
-        # Process API data
-        df = process_weather_data(df)
-        if df is None:
-            return None, "Failed to process data"
-        
-        # Scrape snow depth data and merge with API data
-        snow_df = scrape_snow_depth()
-        if not snow_df.empty:
-            df = df.sort_values('timestamp')
-            snow_df = snow_df.sort_values('timestamp')
-            # Merge using nearest timestamp within a 30-minute tolerance
-            df = pd.merge_asof(df, snow_df, on='timestamp', tolerance=pd.Timedelta('30min'), direction='nearest')
-        
-        print("\nProcessed data sample:")
-        print(df.head())
-        
-        return df, None
-
-    except Exception as e:
-        return None, f"Error: {str(e)}"
+    if "Time" not in headers or "Snow Depth" not in headers:
+        print("Required columns 'Time' or 'Snow Depth' not found in the table headers.")
+        return None
 
-def fetch_and_display(station_id, hours):
-    """
-    Main function to fetch data and create visualization
-    """
-    df, error = get_weather_data(station_id, hours)
+    time_index = headers.index("Time")
+    snow_depth_index = headers.index("Snow Depth")
     
-    if error:
-        return None, error
+    # Extract data rows (skip the header)
+    rows = []
+    for tr in table.find_all("tr")[1:]:
+        cells = tr.find_all("td")
+        # Ensure there are enough cells in the row
+        if len(cells) > max(time_index, snow_depth_index):
+            time_text = cells[time_index].get_text(strip=True)
+            snow_text = cells[snow_depth_index].get_text(strip=True)
+            rows.append((time_text, snow_text))
     
-    if df is not None and not df.empty:
-        fig = create_visualizations(df)
-        return fig, "Data fetched successfully!"
-    
-    return None, "No data available for the specified parameters."
-
-# Create Gradio interface
-with gr.Blocks() as demo:
-    gr.Markdown("# Weather Data Viewer")
-    gr.Markdown("Displays temperature, wind speed, and snow data from NWS stations")
+    # Create a DataFrame from the extracted data
+    df = pd.DataFrame(rows, columns=["Time", "Snow Depth"])
     
-    with gr.Row():
-        station_id = gr.Textbox(label="Station ID", value="YCTIM")
-        hours = gr.Slider(minimum=24, maximum=168, value=72, 
-                         label="Hours of Data", step=24)
+    # Convert the "Time" column to datetime objects
+    df["Time"] = pd.to_datetime(df["Time"], errors="coerce")
     
-    fetch_btn = gr.Button("Fetch Data")
+    # Convert "Snow Depth" to a numeric value (in inches)
+    df["Snow Depth"] = pd.to_numeric(df["Snow Depth"], errors="coerce")
     
-    plot_output = gr.Plot()
-    message = gr.Textbox(label="Status")
-    
-    fetch_btn.click(
-        fn=fetch_and_display,
-        inputs=[station_id, hours],
-        outputs=[plot_output, message]
-    )
+    return df
 
-# Launch the app
-demo.launch()
+if __name__ == "__main__":
+    snow_df = scrape_snow_depth()
+    if snow_df is not None:
+        print("Sample of scraped snow depth data:")
+        print(snow_df.head())
+    else:
+        print("No snow depth data could be scraped.")