Fix: Readme

.env

@@ -1,2 +0,0 @@
-MONGO_URI=mongodb+srv://Agripredict:TjXSvMhOis49qH8E@cluster0.gek7n.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0
-

src/agri_predict/config.py

@@ -19,6 +19,15 @@ def get_collections():
 
     client = MongoClient(mongo_uri, tlsCAFile=certifi.where())
     db = client["AgriPredict"]
+    
+    # Create indexes for better query performance (safe to call multiple times)
+    try:
+        db["WhiteSesame"].create_index([("Reported Date", -1)])
+        db["WhiteSesame"].create_index([("State Name", 1), ("Reported Date", -1)])
+        db["WhiteSesame"].create_index([("Market Name", 1), ("Reported Date", -1)])
+    except Exception:
+        pass  # Indexes might already exist
+    
     return {
         "collection": db["WhiteSesame"],
         "best_params_collection": db["BestParams"],

src/agri_predict/data.py

@@ -7,7 +7,17 @@ from .scraper import api_client
 
 def preprocess_data(df: pd.DataFrame) -> pd.DataFrame:
     df = df[['Reported Date', 'Modal Price (Rs./Quintal)']].copy()
-    df['Reported Date'] = pd.to_datetime(df['Reported Date'])
+    # Ensure datetime and numeric types (may already be converted)
+    if not pd.api.types.is_datetime64_any_dtype(df['Reported Date']):
+        df['Reported Date'] = pd.to_datetime(df['Reported Date'])
+    df['Modal Price (Rs./Quintal)'] = pd.to_numeric(df['Modal Price (Rs./Quintal)'], errors='coerce')
+    
+    # Drop any rows with NaT dates or NaN prices
+    df = df.dropna(subset=['Reported Date', 'Modal Price (Rs./Quintal)'])
+    
+    if df.empty:
+        return df
+    
     df = df.groupby('Reported Date', as_index=False).mean()
     full_date_range = pd.date_range(df['Reported Date'].min(), df['Reported Date'].max())
     df = df.set_index('Reported Date').reindex(full_date_range).rename_axis('Reported Date').reset_index()
@@ -19,14 +29,32 @@ def fetch_and_process_data(query_filter: dict):
     cols = get_collections()
     collection = cols['collection']
     try:
-        cursor = collection.find(query_filter)
+        # Fetch all fields - MongoDB handles this efficiently
+        cursor = collection.find(query_filter).sort('Reported Date', 1)
         data = list(cursor)
-        if data:
-            df = pd.DataFrame(data)
-            df = preprocess_data(df)
-            return df
+        if not data:
+            return None
+            
+        df = pd.DataFrame(data)
+        
+        # Check if required columns exist
+        if 'Reported Date' not in df.columns or 'Modal Price (Rs./Quintal)' not in df.columns:
+            import streamlit as st
+            st.error(f"Missing required columns. Available: {df.columns.tolist()}")
+            return None
+            
+        # Ensure proper data types before preprocessing
+        df['Reported Date'] = pd.to_datetime(df['Reported Date'])
+        df['Modal Price (Rs./Quintal)'] = pd.to_numeric(df['Modal Price (Rs./Quintal)'], errors='coerce')
+        df = preprocess_data(df)
+        
+        if df is None or df.empty:
+            return None
+        return df
+    except Exception as e:
+        import streamlit as st
+        st.error(f"Error fetching data: {str(e)}")
         return None
-    except Exception:
         return None
 
 

src/agri_predict/models.py

@@ -97,7 +97,7 @@ def _train_and_evaluate_generic(df, feature_fn, split_date, progress_bar):
             line=dict(color=color, dash=dash)
         ))
     fig.update_layout(title="Train, Test, and Predicted Data", xaxis_title="Date", yaxis_title="Modal Price (Rs./Quintal)", template="plotly_white")
-    st.plotly_chart(fig, width='stretch')
+    st.plotly_chart(fig, use_container_width=True)
 
     return best_params
 

src/agri_predict/plotting.py

@@ -25,7 +25,7 @@ def plot_data(original_df, future_df, last_date, model, days):
         data = plot_df[plot_df['Type'] == plot_type]
         fig.add_trace(go.Scatter(x=data['Reported Date'], y=data['Modal Price (Rs./Quintal)'], mode='lines', name=f"{plot_type} Data", line=dict(color=color, dash=dash)))
     fig.update_layout(title="Actual vs Forecasted Modal Price (Rs./Quintal)", xaxis_title="Date", yaxis_title="Modal Price (Rs./Quintal)", template="plotly_white")
-    st.plotly_chart(fig, width='stretch')
+    st.plotly_chart(fig, use_container_width=True)
 
 
 def download_button(future_df: pd.DataFrame, key: str):
@@ -87,7 +87,7 @@ def display_statistics(df):
     st.subheader("📆 This Day in Previous Years")
     st.markdown("<p class='highlight'>This table shows the modal price and total arrivals for this exact day across previous years. It provides a historical perspective to compare against current market conditions. This section examines historical data for the same day in previous years. By analyzing trends for this specific day, you can identify seasonal patterns, supply-demand changes, or any deviations that might warrant closer attention.</p>", unsafe_allow_html=True)
     today = latest_date
-    previous_years_data = national_data[national_data['Reported Date'].dt.dayofyear == today.dayofyear]
+    previous_years_data = national_data[national_data['Reported Date'].dt.dayofyear == today.dayofyear].copy()
 
     if not previous_years_data.empty:
         previous_years_data['Year'] = previous_years_data['Reported Date'].dt.year.astype(str)
@@ -121,7 +121,7 @@ def display_statistics(df):
     st.subheader("📈 Largest Daily Price Changes (Past Year)")
     st.markdown("<p class='highlight'>This analysis identifies the most significant daily price changes in the past year. These fluctuations can highlight periods of market volatility, potentially caused by external factors like weather, policy changes, or supply chain disruptions.</p>", unsafe_allow_html=True)
     one_year_ago = latest_date - pd.DateOffset(years=1)
-    recent_data = national_data[national_data['Reported Date'] >= one_year_ago]
+    recent_data = national_data[national_data['Reported Date'] >= one_year_ago].copy()
     recent_data['Daily Change (%)'] = recent_data['Modal Price (Rs./Quintal)'].pct_change() * 100
     largest_changes = recent_data[['Reported Date', 'Modal Price (Rs./Quintal)', 'Daily Change (%)']].nlargest(5, 'Daily Change (%)')
     largest_changes['Reported Date'] = largest_changes['Reported Date'].dt.date
@@ -147,7 +147,7 @@ def display_statistics(df):
     national_data['Lag (14 Days)'] = national_data['Modal Price (Rs./Quintal)'].shift(14)
     national_data['Reported Date'] = national_data['Reported Date'].dt.date
     national_data = national_data.sort_values(by='Reported Date', ascending=False)
-    st.dataframe(national_data.head(14).reset_index(drop=True), width='stretch', height=525)
+    st.dataframe(national_data.head(14).reset_index(drop=True), use_container_width=True, height=525)
 
     editable_spreadsheet()
 

streamlit_app.py

@@ -36,6 +36,39 @@ def get_cached_collections():
     return get_collections()
 
 
+@st.cache_data(ttl=300)  # Cache for 5 minutes
+def load_all_data(_collection):
+    """Load and cache all data from MongoDB collection."""
+    data = list(_collection.find({}, {'_id': 0, 'Reported Date': 1, 'Modal Price (Rs./Quintal)': 1, 'Arrivals (Tonnes)': 1, 'State Name': 1, 'Market Name': 1}))
+    if not data:
+        return pd.DataFrame()
+    df = pd.DataFrame(data)
+    df['Reported Date'] = pd.to_datetime(df['Reported Date'])
+    df['Modal Price (Rs./Quintal)'] = pd.to_numeric(df['Modal Price (Rs./Quintal)'], errors='coerce')
+    df['Arrivals (Tonnes)'] = pd.to_numeric(df['Arrivals (Tonnes)'], errors='coerce')
+    return df
+
+
+@st.cache_data(ttl=300)
+def get_filtered_data(_collection, state=None, market=None, days=30):
+    """Get filtered and cached data based on parameters."""
+    query_filter = {"Reported Date": {"$gte": datetime.now() - timedelta(days=days)}}
+    if state and state != 'India':
+        query_filter["State Name"] = state
+    if market:
+        query_filter["Market Name"] = market
+    
+    data = list(_collection.find(query_filter, {'_id': 0}))
+    if not data:
+        return pd.DataFrame()
+    
+    df = pd.DataFrame(data)
+    df['Reported Date'] = pd.to_datetime(df['Reported Date'])
+    df['Modal Price (Rs./Quintal)'] = pd.to_numeric(df['Modal Price (Rs./Quintal)'], errors='coerce')
+    df['Arrivals (Tonnes)'] = pd.to_numeric(df['Arrivals (Tonnes)'], errors='coerce')
+    return df
+
+
 st.markdown("""
     <style>
         .main { 
@@ -95,15 +128,15 @@ if st.session_state.authenticated:
             query_filter = {}
 
         data_type = st.sidebar.radio("Select Data Type", ["Price", "Volume", "Both"])
-        query_filter["Reported Date"] = {"$gte": datetime.now() - timedelta(days=st.session_state.selected_period)}
 
         if st.sidebar.button("✨ Let's go!"):
             try:
-                cursor = collection.find(query_filter)
-                data = list(cursor)
-                if data:
-                    df = pd.DataFrame(data)
-                    df['Reported Date'] = pd.to_datetime(df['Reported Date'])
+                # Use cached data loading
+                state_param = selected_state if selected_state != 'India' else None
+                market_param = selected_market if market_wise else None
+                df = get_filtered_data(collection, state_param, market_param, st.session_state.selected_period)
+                
+                if not df.empty:
                     df_grouped = df.groupby('Reported Date', as_index=False).agg({'Arrivals (Tonnes)': 'sum', 'Modal Price (Rs./Quintal)': 'mean'})
                     date_range = pd.date_range(start=df_grouped['Reported Date'].min(), end=df_grouped['Reported Date'].max())
                     df_grouped = df_grouped.set_index('Reported Date').reindex(date_range).rename_axis('Reported Date').reset_index()
@@ -118,19 +151,19 @@ if st.session_state.authenticated:
                         fig.add_trace(go.Scatter(x=df_grouped['Reported Date'], y=df_grouped['Scaled Price'], mode='lines', name='Scaled Price', line=dict(width=1, color='green')))
                         fig.add_trace(go.Scatter(x=df_grouped['Reported Date'], y=df_grouped['Scaled Arrivals'], mode='lines', name='Scaled Arrivals', line=dict(width=1, color='blue')))
                         fig.update_layout(title="Price and Arrivals Trend", xaxis_title='Date', yaxis_title='Scaled Values', template='plotly_white')
-                        st.plotly_chart(fig, width='stretch')
+                        st.plotly_chart(fig, use_container_width=True)
                     elif data_type == "Price":
                         import plotly.graph_objects as go
                         fig = go.Figure()
                         fig.add_trace(go.Scatter(x=df_grouped['Reported Date'], y=df_grouped['Modal Price (Rs./Quintal)'], mode='lines', name='Modal Price', line=dict(width=1, color='green')))
                         fig.update_layout(title="Modal Price Trend", xaxis_title='Date', yaxis_title='Price (/Quintall)', template='plotly_white')
-                        st.plotly_chart(fig, width='stretch')
+                        st.plotly_chart(fig, use_container_width=True)
                     else:
                         import plotly.graph_objects as go
                         fig = go.Figure()
                         fig.add_trace(go.Scatter(x=df_grouped['Reported Date'], y=df_grouped['Arrivals (Tonnes)'], mode='lines', name='Arrivals', line=dict(width=1, color='blue')))
                         fig.update_layout(title="Arrivals Trend", xaxis_title='Date', yaxis_title='Volume (in Tonnes)', template='plotly_white')
-                        st.plotly_chart(fig, width='stretch')
+                        st.plotly_chart(fig, use_container_width=True)
                 else:
                     st.warning("⚠️ No data found for the selected filters.")
             except Exception as e:
@@ -147,21 +180,27 @@ if st.session_state.authenticated:
             if not IS_PROD and st.button("Train and Forecast"):
                 query_filter = {"State Name": selected_state}
                 df = fetch_and_process_data(query_filter)
-                if sub_timeline == "14 days":
-                    train_and_forecast(df, filter_key, 14)
-                elif sub_timeline == "1 month":
-                    train_and_forecast(df, filter_key, 30)
+                if df is not None:
+                    if sub_timeline == "14 days":
+                        train_and_forecast(df, filter_key, 14)
+                    elif sub_timeline == "1 month":
+                        train_and_forecast(df, filter_key, 30)
+                    else:
+                        train_and_forecast(df, filter_key, 90)
                 else:
-                    train_and_forecast(df, filter_key, 90)
+                    st.error("❌ No data available for the selected state.")
             if st.button("Forecast"):
                 query_filter = {"State Name": selected_state}
                 df = fetch_and_process_data(query_filter)
-                if sub_timeline == "14 days":
-                    forecast(df, filter_key, 14)
-                elif sub_timeline == "1 month":
-                    forecast(df, filter_key, 30)
+                if df is not None:
+                    if sub_timeline == "14 days":
+                        forecast(df, filter_key, 14)
+                    elif sub_timeline == "1 month":
+                        forecast(df, filter_key, 30)
+                    else:
+                        forecast(df, filter_key, 90)
                 else:
-                    forecast(df, filter_key, 90)
+                    st.error("❌ No data available for the selected state.")
         elif sub_option == "Market":
             market_options = ["Rajkot", "Gondal", "Kalburgi", "Amreli"]
             selected_market = st.selectbox("Select Market for Model Training", market_options)
@@ -169,47 +208,62 @@ if st.session_state.authenticated:
             if not IS_PROD and st.button("Train and Forecast"):
                 query_filter = {"Market Name": selected_market}
                 df = fetch_and_process_data(query_filter)
-                if sub_timeline == "14 days":
-                    train_and_forecast(df, filter_key, 14)
-                elif sub_timeline == "1 month":
-                    train_and_forecast(df, filter_key, 30)
+                if df is not None:
+                    if sub_timeline == "14 days":
+                        train_and_forecast(df, filter_key, 14)
+                    elif sub_timeline == "1 month":
+                        train_and_forecast(df, filter_key, 30)
+                    else:
+                        train_and_forecast(df, filter_key, 90)
                 else:
-                    train_and_forecast(df, filter_key, 90)
+                    st.error("❌ No data available for the selected market.")
             elif st.button("Forecast"):
                 query_filter = {"Market Name": selected_market}
                 df = fetch_and_process_data(query_filter)
-                if sub_timeline == "14 days":
-                    forecast(df, filter_key, 14)
-                elif sub_timeline == "1 month":
-                    forecast(df, filter_key, 30)
+                if df is not None:
+                    if sub_timeline == "14 days":
+                        forecast(df, filter_key, 14)
+                    elif sub_timeline == "1 month":
+                        forecast(df, filter_key, 30)
+                    else:
+                        forecast(df, filter_key, 90)
                 else:
-                    forecast(df, filter_key, 90)
+                    st.error("❌ No data available for the selected market.")
         elif sub_option == "India":
             df = collection_to_dataframe(impExp)
             if not IS_PROD and st.button("Train and Forecast"):
                 query_filter = {}
                 df = fetch_and_process_data(query_filter)
-                if sub_timeline == "14 days":
-                    train_and_forecast(df, "India", 14)
-                elif sub_timeline == "1 month":
-                    train_and_forecast(df, "India", 30)
+                if df is not None:
+                    if sub_timeline == "14 days":
+                        train_and_forecast(df, "India", 14)
+                    elif sub_timeline == "1 month":
+                        train_and_forecast(df, "India", 30)
+                    else:
+                        train_and_forecast(df, "India", 90)
                 else:
-                    train_and_forecast(df, "India", 90)
+                    st.error("❌ No data available for forecasting.")
             if st.button("Forecast"):
                 query_filter = {}
                 df = fetch_and_process_data(query_filter)
-                if sub_timeline == "14 days":
-                    forecast(df, "India", 14)
-                elif sub_timeline == "1 month":
-                    forecast(df, "India", 30)
+                if df is not None:
+                    if sub_timeline == "14 days":
+                        forecast(df, "India", 14)
+                    elif sub_timeline == "1 month":
+                        forecast(df, "India", 30)
+                    else:
+                        forecast(df, "India", 90)
                 else:
-                    forecast(df, "India", 90)
+                    st.error("❌ No data available for forecasting.")
 
     elif view_mode == "Statistics":
-        document = collection.find_one()
-        df = get_dataframe_from_collection(collection)
-        from src.agri_predict.plotting import display_statistics
-        display_statistics(df)
+        # Use cached data loading
+        df = load_all_data(collection)
+        if not df.empty:
+            from src.agri_predict.plotting import display_statistics
+            display_statistics(df)
+        else:
+            st.warning("No data available to display statistics.")
     elif view_mode == "Exim":
         df = collection_to_dataframe(impExp)
         plot_option = st.radio("Select the data to visualize:", ["Import Price", "Import Quantity", "Export Price", "Export Quantity"], horizontal=True)