Create data/app.py

data/app.py

@@ -0,0 +1,427 @@
+import streamlit as st
+import streamlit.components.v1 as components
+import pandas as pd
+import numpy as np
+import joblib
+import plotly.graph_objects as go
+from feature_pipeline import create_features
+from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error
+
+# --- 1. PAGE CONFIGURATION ---
+st.set_page_config(
+    page_title="Group 5 Hanoi Weather Hub",
+    page_icon="☀️",
+    layout="wide"
+)
+
+# --- CONSTANTS ---
+PLOT_COLORS = {'past': '#005aa7', 'forecast': "#1547eb", 'actual': "#d0b51b"}
+
+# --- FEATURE DESCRIPTIONS (50 FEATURES) ---
+FEATURE_DESCRIPTIONS = {
+    'cos_day_of_year': "Cosine of the day of the year. Captures the main seasonal cycle.",
+    'temp': "The average temperature of the current day (°C). The most important feature.",
+    'temp_ewm_14': "14-day Exponentially Weighted Moving Average of temperature.",
+    'season_Winter': "A binary flag (1 if Winter, 0 otherwise).",
+    'winddir_cos': "Cosine of the wind direction. Helps model understand east vs. west wind components.",
+    'temp_ewm_7': "7-day Exponentially Weighted Moving Average of temperature.",
+    'temp_ewm_30': "30-day Exponentially Weighted Moving Average of temperature.",
+    'temp_diff_1': "Difference between today's and yesterday's temperature.",
+    'sin_day_of_year': "Sine of the day of the year. Pinpoints the exact time of year.",
+    'solarradiation': "Amount of solar energy received. A primary driver of heat.",
+    'windgust': "The highest instantaneous wind speed recorded during the day (km/h).",
+    'windspeed': "The average wind speed for the day (km/h).",
+    'wind_vector_ew': "East-West wind vector component, calculated from wind speed and direction.",
+    'temp_diff_7': "The difference in temperature compared to 7 days ago.",
+    'fft_sin_freq_1': "A cyclical feature from Fourier analysis for a dominant cycle.",
+    'temp_roll_min_3': "The minimum temperature recorded over the past 3 days.",
+    'fft_sin_freq_2': "A cyclical feature for the second most dominant cycle.",
+    'humidity_lag_9': "Humidity from 9 days ago.",
+    'windspeed_diff_7': "The difference in wind speed compared to 7 days ago.",
+    'humidity_ewm_30': "30-day Exponentially Weighted Moving Average of humidity.",
+    'day_of_month': "The day of the month (1-31).",
+    'temp_roll_min_7': "The minimum temperature recorded over the past 7 days.",
+    'precip_ewm_30': "30-day Exponentially Weighted Moving Average of precipitation.",
+    'humidity_ewm_14': "14-day Exponentially Weighted Moving Average of humidity.",
+    'humidity': "The relative humidity of the air (%).",
+    'temp_range': "Difference between the daily max (tempmax) and min (tempmin) temperature.",
+    'temp_resid_yearly': "The residual (noise) component from temperature's seasonal decomposition.",
+    'humidity_roll_std_7': "7-day rolling standard deviation of humidity.",
+    'fft_cos_freq_1': "A cyclical feature paired with fft_sin_freq_1.",
+    'windspeed_roll_min_30': "The minimum wind speed recorded in the past 30 days.",
+    'precipcover': "Percentage of the day with precipitation cover.",
+    'windspeed_lag_7': "Wind speed from 7 days ago.",
+    'precip_diff_1': "Difference between today's and yesterday's precipitation.",
+    'season_Spring': "A binary flag (1 if Spring, 0 otherwise).",
+    'precip_lag_3': "Precipitation from 3 days ago.",
+    'temp_lag_1': "The average temperature from the previous day.",
+    'precip_diff_7': "The difference in precipitation compared to 7 days ago.",
+    'precip': "The amount of precipitation (rain) recorded (mm).",
+    'cloudcover_roll_std_3': "3-day rolling standard deviation of cloud cover.",
+    'season_Fall': "A binary flag (1 if Fall, 0 otherwise).",
+    'windspeed_ewm_30': "30-day Exponentially Weighted Moving Average of wind speed.",
+    'precip_roll_min_3': "The minimum precipitation recorded in the past 3 days.",
+    'cloudcover_roll_std_30': "30-day rolling standard deviation of cloud cover.",
+    'precipprob': "The probability of precipitation occurring (%).",
+    'temp_roll_std_3': "3-day rolling standard deviation of temperature.",
+    'humidity_diff_7': "The difference in humidity compared to 7 days ago.",
+    'windspeed_roll_std_3': "3-day rolling standard deviation of wind speed.",
+    'windspeed_lag_3': "Wind speed from 3 days ago.",
+    'humidity_roll_std_3': "3-day rolling standard deviation of humidity.",
+    'icon_clear-day': "A binary flag if the weather icon was 'clear-day'."
+}
+
+# --- 2. CUSTOM CSS FOR STYLING ---
+def load_css():
+    # This block contains your original CSS to keep the UI consistent.
+    st.markdown("""
+    <style>
+    /* ===== Background gradient ===== */
+    [data-testid="stAppViewContainer"] {
+    background-image: linear-gradient(to bottom, #d4e1da 20%, #005aa7 100%) !important;
+    background-attachment: fixed !important;
+    background-size: cover !important;
+    }
+    /* ===== Global text ===== */
+    .stApp, h1, h2, h3, p, label {
+    color: #0A1931;
+    }
+    /* ===== Tabs ===== */
+    button[data-baseweb="tab"][aria-selected="true"] {
+    background-color: #FFFFFF !important;
+    color: #005aa7 !important;
+    padding: 10px 16px !important;
+    border-radius: 10px 10px 0 0 !important;
+    font-weight: 700 !important;
+    border-bottom: 3px solid #e53935 !important;
+    }
+    button[data-baseweb="tab"][aria-selected="false"] {
+    background-color: rgba(255,255,255,0.3) !important;
+    color: #284270 !important;
+    padding: 10px 16px !important;
+    }
+    /* ===== Metrics: Forecast titles, main numbers, and Actual (delta) ===== */
+    [data-testid="stMetricLabel"] p,
+    div[data-testid="stMetricValue"] {
+    font-weight: 700 !important;
+    text-shadow: 1px 1px 3px rgba(0,0,0,0.3);
+    }
+    div[data-testid="stMetricLabel"] p:contains("Forecast for") {
+    color: #ffffff !important;
+    font-size: 2.2rem !important;
+    font-weight: 800 !important;
+    text-shadow: 1px 1px 3px rgba(0,0,0,0.3);
+    background: none !important;
+    border: none !important;
+    }
+    div[data-testid="stMetricValue"] {
+    color: #fefefe !important;
+    font-size: 1.9rem !important;
+    text-shadow: 1px 1px 6px rgba(0,0,0,0.5);
+    }
+    div[data-testid="stMetricLabel"] p {
+    color: #dce9ff !important;
+    font-weight: 700 !important;
+    letter-spacing: 0.3px;
+    }
+    div[data-testid="stMetricLabel"] p {
+    color: #e8f1ff !important;
+    font-size: 1.4rem !important;
+    font-weight: 700 !important;
+    text-shadow: none !important;
+    background: rgba(0, 0, 0, 0.15);
+    padding: 4px 8px;
+    border-radius: 6px;
+    }
+    div[data-testid="stMetricValue"] {
+    color: #ffffff !important;
+    font-size: 1.8rem !important;
+    text-shadow: 0 0 3px rgba(0,0,0,0.3);
+    background: rgba(0,0,0,0.15);
+    border-radius: 6px;
+    padding: 6px 10px;
+    }
+    div[data-testid="stMetricValue"] {
+    color: #fefefe !important;
+    font-size: 1.4rem !important;
+    text-shadow: 1px 1px 6px rgba(0,0,0,0.5);
+    }
+    div[data-testid="stMetricLabel"] p {
+    color: #dce9ff !important;
+    font-weight: 700 !important;
+    letter-spacing: 0.3px;
+    }
+    div[data-testid="stMetricValue"],
+    div[data-testid="stMetricLabel"] p {
+    filter: drop-shadow(0 0 3px rgba(255,255,255,0.4));
+    }
+    /* ===== Custom feature cards (Tab 2) ===== */
+    .custom-card-transparent {
+    background: linear-gradient(145deg, rgba(255,255,255,0.12), rgba(0,0,0,0.25));
+    border: 1px solid rgba(255,255,255,0.25);
+    border-radius: 12px;
+    padding: 14px;
+    text-align: center;
+    transition: all 0.25s ease;
+    box-shadow: 0 2px 8px rgba(0,0,0,0.15);
+    width: 95%;
+    margin: 0 auto;
+    }
+    .custom-card-transparent:hover {
+    transform: translateY(-3px);
+    box-shadow: 0 4px 12px rgba(0,0,0,0.25);
+    filter: brightness(1.1);
+    }
+    .custom-card-transparent h2 {
+    color: #fefefe !important;
+    font-size: 1.6rem !important;
+    font-weight: 700 !important;
+    text-shadow: 0 0 4px rgba(255,255,255,0.25);
+    margin-top: 10px;
+    }
+    .feature-tag-final {
+    background-color: #1c3d70 !important;
+    padding: 5px 10px;
+    border-radius: 8px;
+    font-size: 0.9rem;
+    font-weight: 600;
+    color: #e3eeff !important;
+    text-shadow: 0 0 2px rgba(255,255,255,0.15);
+    display: inline-block;
+    margin-bottom: 3px;
+    }
+    /* ===== Restored missing pieces ===== */
+    div[data-testid="stExpander"] {
+    background-color: rgba(255, 255, 255, 0.85) !important;
+    border-radius: 12px !important;
+    border: 1px solid rgba(0, 90, 167, 0.25) !important;
+    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1) !important;
+    margin-top: 2rem;
+    }
+    div.st-emotion-cache-1e5k5x7 {
+    background-color: rgba(230, 240, 255, 0.5) !important;
+    border-radius: 15px !important;
+    padding: 1rem 0.5rem !important;
+    border: 1px solid rgba(255, 255, 255, 0.3);
+    box-shadow: 0 4px 12px rgba(0,0,0,0.1);
+    margin: 1rem 0;
+    }
+    div.st-emotion-cache-1e5k5x7 div[data-testid="stMetricValue"] {
+    color: white !important;
+    text-shadow: 1px 1px 3px rgba(0,0,0,0.4);
+    }
+    div.st-emotion-cache-1e5k5x7 div[data-testid="stMetricLabel"] p {
+    color: #0A1931 !important;
+    font-weight: 600 !important;
+    }
+    button[data-baseweb="tab"][aria-selected="true"] {
+    border-bottom: 2px solid rgba(229,57,53,0.9) !important;
+    }
+    div[data-testid="stMetricLabel"] p:contains("Forecast for") {
+    color: #ffffff !important;
+    font-size: 2.2rem !important;
+    font-weight: 800 !important;
+    text-shadow: 1px 1px 3px rgba(0,0,0,0.3);
+    background: none !important;
+    border: none !important;
+    }
+    div[data-testid="stMetricValue"] {
+    color: #ffffff !important;
+    font-size: 2.2rem !important;
+    font-weight: 800 !important;
+    text-shadow: 2px 2px 5px rgba(0,0,0,0.4);
+    background: none !important;
+    border: none !important;
+    padding: 0 !important;
+    }
+    [data-testid="stMetricDelta"] {
+    font-size: 1.2rem !important;
+    font-weight: 700 !important;
+    color: #ffea7a !important;
+    text-shadow: 1px 1px 3px rgba(0,0,0,0.4);
+    }
+    div[data-testid="stMetricLabel"] p {
+    font-size: 1.5rem !important;
+    font-weight: 700 !important;
+    color: #06285a !important;
+    text-shadow: none !important;
+    background: none !important;
+    }
+    div[data-testid="stMetricValue"],
+    div[data-testid="stMetricLabel"] p {
+    filter: none !important;
+    }
+    /* ===== Custom Styling for Date INPUT FIELD ONLY ===== */
+    div[data-testid="stDateInput"] input {
+    background-color: #f0f2f6 !important;
+    color: #0A1931 !important;
+    border: 0.5px solid #cccccc !important;
+    border-radius: 4px;
+    padding: 8px 10px;
+    box-shadow: inset 0 1px 2px rgba(75, 98, 138);
+    transition: border-color 0.2s ease, box-shadow 0.2s ease;
+    }
+    div[data-testid="stDateInput"] button {
+    display: none !important;
+    }
+    div[data-testid="stDateInput"] input:hover,
+    div[data-testid="stDateInput"] input:focus {
+    border-color: #1547eb !important;
+    box-shadow: inset 0 1px 1px rgba(0,0,0,0.1), 0 0 0 2px rgba(21, 71, 235, 0.2);
+    }
+    </style>
+    """, unsafe_allow_html=True)
+
+# --- 3. DATA & MODEL LOADING ---
+@st.cache_data
+def load_data_and_artifacts():
+    raw_df = pd.read_csv('data/Hanoi Daily 10 years.csv')
+    y_test_df = pd.read_csv('data/y_test.csv', index_col='datetime', parse_dates=True)
+    feature_importances = pd.read_csv('artifacts/feature_importances.csv')
+    return raw_df, y_test_df, feature_importances
+
+@st.cache_resource
+def load_model_and_features():
+    model = joblib.load('artifacts/hanoi_temp_predictor.joblib')
+    selected_features = joblib.load('artifacts/selected_features.joblib')
+    return model, selected_features
+
+@st.cache_data
+def get_processed_data(_raw_df):
+    return create_features(_raw_df)
+
+# --- 4. MAIN APP LOGIC ---
+load_css()
+st.title("☀️ Group 5 Hanoi Weather Hub")
+st.write("An interactive dashboard to forecast Hanoi's temperature for the next 5 days.")
+
+raw_df, y_test, feature_importances = load_data_and_artifacts()
+model, selected_features = load_model_and_features()
+
+with st.spinner("Running feature engineering pipeline... This may take a moment."):
+    processed_df = get_processed_data(raw_df)
+
+col1, col2 = st.columns([1, 2])
+with col1:
+    selected_date = st.date_input(
+        label="Select a date to forecast from:",
+        value=processed_df.index.max(),
+        min_value=processed_df.index.min(),
+        max_value=processed_df.index.max(),
+        format="YYYY-MM-DD"
+    )
+
+tab1, tab2, tab3 = st.tabs(["📈 Interactive Forecast", "🧠 Forecast Deep Dive", "📊 Model Performance"])
+
+with tab1:
+    selected_date_ts = pd.Timestamp(selected_date)
+    st.header(f"5-Day Forecast from {selected_date_ts.strftime('%Y-%m-%d')}")
+    
+    # Check if selected date exists in the processed dataframe
+    if selected_date_ts not in processed_df.index:
+        st.error("Selected date is not available in the dataset after processing. Please choose a different date.")
+    else:
+        input_data = processed_df.loc[[selected_date_ts]]
+        input_features = input_data[selected_features]
+        prediction = model.predict(input_features)[0]
+        actual_values = []
+        is_in_test_set = selected_date_ts in y_test.index
+        
+        if is_in_test_set:
+            st.info("This is a backtest. 'Actual' values are shown for comparison.", icon="ℹ️")
+            actual_values = y_test.loc[selected_date_ts].values
+        else:
+            st.warning("This is a live forecast. 'Actual' values are not yet available.", icon="⚠️")
+        
+        pred_cols = st.columns(5)
+        forecast_dates = pd.date_range(start=selected_date_ts, periods=6, freq='D')[1:]
+        for i, date in enumerate(forecast_dates):
+            delta_text = ""
+            if is_in_test_set and len(actual_values) > i:
+                delta_text = f"Actual: {actual_values[i]:.1f}°C"
+            pred_cols[i].metric(label=f"Forecast for {date.strftime('%b %d')}", value=f"{prediction[i]:.1f}°C", delta=delta_text, delta_color="off")
+        
+        st.subheader("Visualizations")
+        st.markdown("#### Historical Context: Past 30 Days")
+        hist_data = processed_df.loc[selected_date_ts - pd.Timedelta(days=30):selected_date_ts, 'temp']
+        fig_hist = go.Figure()
+        fig_hist.add_trace(go.Scatter(x=hist_data.index, y=hist_data, mode='lines', name='Past 30 Days', line=dict(color='#636EFA')))
+        fig_hist.update_layout(title={'text': "<b>Actual Temperature - Past 30 Days</b>", 'x': 0.5}, xaxis_title="Date", yaxis_title="Temperature (°C)", paper_bgcolor="#fafbfc", plot_bgcolor="#e5ecf6")
+        components.html(fig_hist.to_html(include_plotlyjs='cdn'), height=450)
+        
+        if is_in_test_set:
+            st.markdown("#### Forecast vs. Actual Comparison")
+            fig_comp = go.Figure()
+            fig_comp.add_trace(go.Scatter(x=forecast_dates, y=prediction, mode='lines+markers', name='5-Day Forecast', line=dict(color=PLOT_COLORS['forecast'])))
+            if len(actual_values) > 0:
+                fig_comp.add_trace(go.Scatter(x=forecast_dates, y=actual_values, mode='lines+markers', name='5-Day Actual', line=dict(color=PLOT_COLORS['actual'])))
+            fig_comp.update_layout(title={'text': "<b>5-Day Forecast vs. Actual Temperature</b>", 'x': 0.5}, xaxis_title="Date", yaxis_title="Temperature (°C)", paper_bgcolor='#fafbfc', plot_bgcolor='#e5ecf6')
+            components.html(fig_comp.to_html(include_plotlyjs='cdn'), height=450)
+
+with tab2:
+    st.header("What were the most important factors for this forecast?")
+    st.write(f"For the forecast made on {selected_date.strftime('%Y-%m-%d')}, the model paid most attention to these factors:")
+    
+    top_5_features = feature_importances.head(5)['Feature'].tolist()
+    key_factor_cols = st.columns(len(top_5_features))
+    
+    for i, feature in enumerate(top_5_features):
+        value = processed_df.loc[pd.Timestamp(selected_date), feature]
+        with key_factor_cols[i]:
+            st.markdown(f"""
+            <div class="custom-card-transparent">
+                <span class="feature-tag-final">{feature}</span>
+                <h2>{value:.2f}</h2>
+            </div>
+            """, unsafe_allow_html=True)
+    
+    st.markdown("<br>", unsafe_allow_html=True)
+    st.subheader("Overall Feature Importance")
+    top_10_df = feature_importances.head(10)
+    fig_imp = go.Figure(go.Bar(
+        x=top_10_df['Importance_Mean'], 
+        y=top_10_df['Feature'], 
+        orientation='h', 
+        marker_color='#005aa7'
+    ))
+    fig_imp.update_layout(
+        title={'text': '<b>Top 10 Most Important Features (Overall)</b>', 'x': 0.5},
+        xaxis_title='Permutation Importance Score',
+        yaxis={'title': '', 'autorange': "reversed"},
+        margin=dict(l=150, r=20, t=50, b=70),
+        paper_bgcolor='#fafbfc', plot_bgcolor='#e5ecf6'
+    )
+    components.html(fig_imp.to_html(include_plotlyjs='cdn'), height=520)
+    st.markdown("<br>", unsafe_allow_html=True)
+
+    st.header("Feature Glossary")
+    with st.expander(f"Click to learn about all {len(FEATURE_DESCRIPTIONS)} model features", expanded=False):
+        glossary_df = pd.DataFrame(
+            FEATURE_DESCRIPTIONS.items(), 
+            columns=['Feature', 'Description']
+        ).sort_values(by='Feature').reset_index(drop=True)
+        st.table(glossary_df)
+        
+with tab3:
+    st.header("Model Performance on the Entire Test Set")
+    X_test_filtered = processed_df.loc[y_test.index][selected_features]
+    y_pred_test = model.predict(X_test_filtered)
+    
+    macro_r2 = r2_score(y_test, y_pred_test)
+    macro_rmse = np.sqrt(mean_squared_error(y_test, y_pred_test))
+    macro_mae = mean_absolute_error(y_test, y_pred_test)
+    
+    m_cols = st.columns(3)
+    m_cols[0].metric("Average R2 Score", f"{macro_r2:.3f}")
+    m_cols[1].metric("Average RMSE", f"{macro_rmse:.3f}°C")
+    m_cols[2].metric("Average MAE", f"{macro_mae:.3f}°C")
+    
+    st.subheader("Prediction vs. Actual Values")
+    y_test_flat = y_test.values.flatten()
+    y_pred_flat = y_pred_test.flatten()
+    
+    fig_scatter = go.Figure()
+    fig_scatter.add_trace(go.Scatter(x=y_test_flat, y=y_pred_flat, mode='markers', marker=dict(color="rgba(73, 82, 199, 0.6)"), name='Predictions'))
+    fig_scatter.add_trace(go.Scatter(x=[y_test_flat.min(), y_test_flat.max()], y=[y_test_flat.min(), y_test_flat.max()], mode='lines', line=dict(color='#EF553B', dash='dash'), name='Perfect Prediction Line'))
+    fig_scatter.update_layout(title={'text': "<b>How well do predictions match actual values?</b>", 'x': 0.5}, xaxis_title="Actual Temperature (°C)", yaxis_title="Predicted Temperature (°C)", paper_bgcolor='#fafbfc', plot_bgcolor='#e5ecf6')
+    components.html(fig_scatter.to_html(include_plotlyjs='cdn'), height=600)