Update app.py

app.py

@@ -35,6 +35,7 @@ else:
     best_params_collection_3m = db["BestParams_3m"]
     impExp = db["impExp"]
     users_collection = db["user"]
+    market_price_data = db["new_data_price"]
 
 state_market_dict = {
     "Karnataka": [
@@ -691,7 +692,7 @@ def forecast_next_14_days(df, _best_params, key):
     future_df['Modal Price (Rs./Quintal)'] = future_predictions
 
     # Pass model to plot_data
-    plot_data(original_df, future_df, last_date, model, 14)
+    plot_data(original_df, future_df, last_date, 14)
     download_button(future_df, key)
     
 def forecast_next_30_days(df, _best_params, key):
@@ -717,7 +718,7 @@ def forecast_next_30_days(df, _best_params, key):
     future_df['Modal Price (Rs./Quintal)'] = future_predictions
 
     # Pass model to plot_data
-    plot_data(original_df, future_df, last_date, model, 30)
+    plot_data(original_df, future_df, last_date, 30)
     download_button(future_df, key)
 
 def forecast_next_90_days(df, _best_params, key):
@@ -743,30 +744,48 @@ def forecast_next_90_days(df, _best_params, key):
     future_df['Modal Price (Rs./Quintal)'] = future_predictions
 
     # Pass model to plot_data
-    plot_data(original_df, future_df, last_date, model, 90)
+    plot_data(original_df, future_df, last_date, 90)
     download_button(future_df, key)
 
-def plot_data(original_df, future_df, last_date, model, days):
-    actual_last_df = original_df[original_df['Reported Date'] > (last_date - pd.Timedelta(days=days))]
-    predicted_plot_df = actual_last_df[['Reported Date']].copy()
-    predicted_plot_df['Modal Price (Rs./Quintal)'] = model.predict(
-        actual_last_df.drop(columns=['Modal Price (Rs./Quintal)', 'Reported Date'], errors='ignore'))
-    predicted_plot_df['Type'] = 'Actual'
-
+def plot_data(original_df, future_df, last_date, days):
+    # Filter original_df for the period you want to plot.
+    actual_df = original_df[original_df['Reported Date'] >= (last_date - pd.Timedelta(days=days))].copy()
+    actual_df['Type'] = 'Actual'
+    
+    # Prepare the future_df (predicted data) and mark it as forecasted.
     future_plot_df = future_df[['Reported Date', 'Modal Price (Rs./Quintal)']].copy()
     future_plot_df['Type'] = 'Forecasted'
-    last_actual_point = predicted_plot_df.iloc[[-1]].copy()
-    last_actual_point['Type'] = 'Forecasted'
+    
+    # Get the last actual data point from actual_df.
+    # Ensure the DataFrame is sorted by date.
+    last_actual_point = actual_df.sort_values('Reported Date').iloc[[-1]].copy()
     future_plot_df = pd.concat([last_actual_point, future_plot_df])
-    plot_df = pd.concat([predicted_plot_df, future_plot_df])
-
+    
+    # Combine both actual and forecasted data for plotting.
+    plot_df = pd.concat([actual_df, future_plot_df])
+    
+    # Create the plot.
     fig = go.Figure()
     for plot_type, color, dash in [('Actual', 'blue', 'solid'), ('Forecasted', 'red', 'dash')]:
         data = plot_df[plot_df['Type'] == plot_type]
-        fig.add_trace(go.Scatter(x=data['Reported Date'], y=data['Modal Price (Rs./Quintal)'], mode='lines', name=f"{plot_type} Data", line=dict(color=color, dash=dash)))
-    fig.update_layout(title="Actual vs Forecasted Modal Price (Rs./Quintal)", xaxis_title="Date", yaxis_title="Modal Price (Rs./Quintal)", template="plotly_white")
+        fig.add_trace(go.Scatter(
+            x=data['Reported Date'],
+            y=data['Modal Price (Rs./Quintal)'],
+            mode='lines',
+            name=f"{plot_type} Data",
+            line=dict(color=color, dash=dash)
+        ))
+    
+    fig.update_layout(
+        title="Actual vs Forecasted Modal Price (Rs./Quintal)",
+        xaxis_title="Date",
+        yaxis_title="Modal Price (Rs./Quintal)",
+        template="plotly_white"
+    )
+    
     st.plotly_chart(fig, use_container_width=True)
 
+
 def download_button(future_df, key):
     # Create a new DataFrame with only 'Reported Date' and 'Modal Price (Rs./Quintal)'
     download_df = future_df[['Reported Date', 'Modal Price (Rs./Quintal)']].copy()
@@ -787,7 +806,7 @@ def download_button(future_df, key):
 
 
 
-def fetch_and_process_data(query_filter):
+def fetch_and_process_data(query_filter, collection):
     try:
         cursor = collection.find(query_filter)
         data = list(cursor)
@@ -800,7 +819,7 @@ def fetch_and_process_data(query_filter):
             st.warning("⚠️ No data found for the selected filter.")
             return None
     except Exception as e:
-        st.error(f"❌ Error fetching data: {e}")
+        st.error(f"❌ Error fetching data 1: {e}")
         return None
 
 def save_best_params(collection, filter_key, best_params):
@@ -1056,7 +1075,11 @@ def fetch_and_store_data():
         from_date = "01 Jan 2000"
 
     to_date = (datetime.now() - timedelta(days=1)).strftime('%d %b %Y')
-
+    from_date_obj = datetime.strptime(from_date, '%d %b %Y')
+    to_date_obj = datetime.strptime(to_date, '%d %b %Y')
+    if to_date_obj < from_date_obj:
+        print("Data already scraped")
+        return None
     # Build the URL to be requested
     base_url = "https://agmarknet.gov.in/SearchCmmMkt.aspx"
     params = {
@@ -1088,15 +1111,10 @@ def fetch_and_store_data():
     if response.status_code == 200:
         soup = BeautifulSoup(response.content, 'html.parser')
         table = soup.find("table", {"class": "tableagmark_new"})
-
+        st.write(soup.prettify())
         if table:
             headers = [th.get_text(strip=True) for th in table.find_all("th")]
-            rows = []
-
-            for row in table.find_all("tr")[1:]:
-                cells = [td.get_text(strip=True) for td in row.find_all("td")]
-                if cells:
-                    rows.append(cells)
+            rows = [[td.get_text(strip=True) for td in row.find_all("td")] for row in table.find_all("tr")[1:]]
 
             df = pd.DataFrame(rows, columns=headers)
             df = df[df['Variety']=="White"]
@@ -1110,11 +1128,98 @@ def fetch_and_store_data():
             df["Arrivals (Tonnes)"] = pd.to_numeric(df["Arrivals (Tonnes)"], errors='coerce').astype("float64")
             df["state"] = df["state"].astype("string")
             df["Market Name"] = df["Market Name"].astype("string")
+            records = df.to_dict(orient="records")
+            if records:
+                collection.insert_many(records)
+                print(f"Inserted {len(records)} new records into MongoDB.")
+            else:
+                print("No new records to insert.")
+
+            return df
+
+    else:
+        print(f"Failed to fetch data with status code: {response.status_code}")
+        return None
+        
+
+def fetch_and_store_data_market():
+    latest_doc = market_price_data.find_one(sort=[("Reported Date", -1)])
+    if latest_doc and "Reported Date" in latest_doc:
+        latest_date = latest_doc["Reported Date"]
+    else:
+        latest_date = None
+
+    if latest_date:
+        from_date = (latest_date + timedelta(days=1)).strftime('%d %b %Y')
+    else:
+        # If no latest date, set a default from_date
+        from_date = "01 Jan 2000"
+
+    to_date = (datetime.now() - timedelta(days=1)).strftime('%d %b %Y')
+    from_date_obj = datetime.strptime(from_date, '%d %b %Y')
+    to_date_obj = datetime.strptime(to_date, '%d %b %Y')
+    if to_date_obj < from_date_obj:
+        print("Data already scraped")
+        return None
+    # Build the URL to be requested
+    base_url = "https://agmarknet.gov.in/SearchCmmMkt.aspx"
+    params = {
+        "Tx_Commodity": "11",
+        "Tx_State": "0",
+        "Tx_District": "0",
+        "Tx_Market": "0",
+        "DateFrom": from_date,
+        "DateTo": to_date,
+        "Fr_Date": from_date,
+        "To_Date": to_date,
+        "Tx_Trend": "0",
+        "Tx_CommodityHead": "Sesamum(Sesame,Gingelly,Til)",
+        "Tx_StateHead": "--Select--",
+        "Tx_DistrictHead": "--Select--",
+        "Tx_MarketHead": "--Select--"
+        }
+    
+    full_url = f"{base_url}?{'&'.join(f'{k}={v}' for k, v in params.items())}"
+    api_url = "https://api.scraperapi.com"
+    api_key = "bbbbde6b56c0fde1e2a61c914eb22d14"
+    scraperapi_params = {
+        'api_key': api_key,
+        'url': full_url
+    }
+    st.write(full_url)
+
+    response = requests.get(api_url, params=scraperapi_params)
+
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.content, 'html.parser')
+        table = soup.find("table", {"class": "tableagmark_new"})
 
-            for index, row in df.iterrows():
-                document = row.to_dict()
-                collection.insert_one(document)
+        if table:
+            headers = [th.get_text(strip=True) for th in table.find_all("th")]
+            rows = []
 
+            for row in table.find_all("tr")[1:]:
+                cells = [td.get_text(strip=True) for td in row.find_all("td")]
+                if cells:
+                    rows.append(cells)
+            st.write(rows)
+            st.write(headers)
+            df = pd.DataFrame(rows, columns=headers)
+            df = df[df['Variety']=="White"]
+            df["Price Date"] = pd.to_datetime(df["Price Date"], format='%d %b %Y', errors='coerce')
+            df.dropna(subset=["Price Date"], inplace=True)
+            df.sort_values(by="Price Date", inplace=True)
+            df = df[df["Grade"]=="FAQ"]
+            # Type casting for the columns
+            df["Modal Price (Rs./Quintal)"] = pd.to_numeric(df["Modal Price (Rs./Quintal)"], errors='coerce').astype("int64")
+            df["Market Name"] = df["Market Name"].astype("string")
+            df.rename(columns={"Price Date": "Reported Date"}, inplace=True)
+            records = df.to_dict(orient="records")
+            if records:
+                collection.insert_many(records)
+                print(f"Inserted {len(records)} new records into MongoDB.")
+            else:
+                print("No new records to insert.")
             return df
 
     else:
@@ -1124,13 +1229,8 @@ def fetch_and_store_data():
 
     
 def get_dataframe_from_collection(collection):
-    # Fetch all documents from the collection
     data = list(collection.find())
-
-    # Convert the list of documents into a DataFrame
     df = pd.DataFrame(data)
-
-    # Drop the MongoDB-specific '_id' column (optional, if not needed)
     if "_id" in df.columns:
         df = df.drop(columns=["_id"])
 
@@ -1142,7 +1242,6 @@ def authenticate_user(username, password):
         return True
     return False
 
-# CSS for responsive and professional design
 st.markdown("""
     <style>
         /* Main layout adjustments */
@@ -1222,6 +1321,7 @@ if st.session_state.authenticated:
     st.title("🌾 AgriPredict Dashboard")
     if st.button("Get Live Data Feed"):
         fetch_and_store_data()
+        fetch_and_store_data_market()
     # Top-level radio buttons for switching views
     view_mode = st.radio("", ["Statistics", "Plots", "Predictions", "Exim"], horizontal=True)
 
@@ -1252,7 +1352,7 @@ if st.session_state.authenticated:
             if market_wise:
                 markets = state_market_dict.get(selected_state, [])
                 selected_market = st.sidebar.selectbox("Select Market", markets)
-                query_filter = {"state": selected_state, "Market Name": selected_market}
+                query_filter = {"Market Name": selected_market}
             else:
                 query_filter = {"state": selected_state}
         else:
@@ -1271,8 +1371,23 @@ if st.session_state.authenticated:
         
         # Submit button to trigger the query and plot
         if st.sidebar.button("✨ Let's go!"):
-            # Fetch data from MongoDB
             try:
+                df_market_grouped = []
+                if "Market Name" in query_filter:
+                    market_cursor = market_price_data.find(query_filter)
+                    market_data = list(market_cursor)
+                    df_market = pd.DataFrame(market_data)
+                    df_market_grouped = df_market.groupby('Reported Date', as_index=False).agg({
+                        'Modal Price (Rs./Quintal)': 'mean'
+                    })
+                    date_range = pd.date_range(
+                        start=df_market_grouped['Reported Date'].min(),
+                        end=df_market_grouped['Reported Date'].max()
+                    )
+                    df_market_grouped = df_market_grouped.set_index('Reported Date').reindex(date_range).rename_axis('Reported Date').reset_index()
+                    df_market_grouped['Modal Price (Rs./Quintal)'] = df_market_grouped['Modal Price (Rs./Quintal)'].fillna(method='ffill').fillna(method='bfill')
+                    
+                    
                 cursor = collection.find(query_filter)
                 data = list(cursor)
         
@@ -1313,16 +1428,20 @@ if st.session_state.authenticated:
                         df_grouped[['Scaled Price', 'Scaled Arrivals']] = scaler.fit_transform(
                             df_grouped[['Modal Price (Rs./Quintal)', 'Arrivals (Tonnes)']]
                         )
+                        if market_data!=[]:
+                            df_market_grouped['Scaled Price'] = scaler.fit_transform(
+                                df_market_grouped[["Modal Price (Rs./Quintal)"]]
+                            ) 
         
                         fig = go.Figure()
         
                         fig.add_trace(go.Scatter(
-                            x=df_grouped['Reported Date'],
-                            y=df_grouped['Scaled Price'],
+                            x=df_market_grouped['Reported Date'],
+                            y=df_market_grouped['Scaled Price'],
                             mode='lines',
                             name='Scaled Price',
                             line=dict(width=1, color='green'),
-                            text=df_grouped['Modal Price (Rs./Quintal)'],
+                            text=df_market_grouped['Modal Price (Rs./Quintal)'],
                             hovertemplate='Date: %{x}<br>Scaled Price: %{y:.2f}<br>Actual Price: %{text:.2f}<extra></extra>'
                         ))
         
@@ -1346,16 +1465,28 @@ if st.session_state.authenticated:
         
                     elif data_type == "Price":
                         # Plot Modal Price
-                        fig = go.Figure()
-                        fig.add_trace(go.Scatter(
-                            x=df_grouped['Reported Date'],
-                            y=df_grouped['Modal Price (Rs./Quintal)'],
-                            mode='lines',
-                            name='Modal Price',
-                            line=dict(width=1, color='green')
-                        ))
-                        fig.update_layout(title="Modal Price Trend", xaxis_title='Date', yaxis_title='Price (/Quintall)', template='plotly_white')
-                        st.plotly_chart(fig, use_container_width=True)
+                        if "Market Name" in query_filter:
+                            fig = go.Figure()
+                            fig.add_trace(go.Scatter(
+                                x=df_market_grouped['Reported Date'],
+                                y=df_market_grouped['Modal Price (Rs./Quintal)'],
+                                mode='lines',
+                                name='Modal Price',
+                                line=dict(width=1, color='green')
+                            ))
+                            fig.update_layout(title="Modal Price Trend", xaxis_title='Date', yaxis_title='Price (/Quintall)', template='plotly_white')
+                            st.plotly_chart(fig, use_container_width=True)
+                        else:
+                            fig = go.Figure()
+                            fig.add_trace(go.Scatter(
+                                x=df_grouped['Reported Date'],
+                                y=df_grouped['Modal Price (Rs./Quintal)'],
+                                mode='lines',
+                                name='Modal Price',
+                                line=dict(width=1, color='green')
+                            ))
+                            fig.update_layout(title="Modal Price Trend", xaxis_title='Date', yaxis_title='Price (/Quintall)', template='plotly_white')
+                            st.plotly_chart(fig, use_container_width=True)                            
         
                     elif data_type == "Volume":
                         # Plot Arrivals (Tonnes)
@@ -1374,7 +1505,7 @@ if st.session_state.authenticated:
                     st.warning("⚠️ No data found for the selected filters.")
         
             except Exception as e:
-                st.error(f"❌ Error fetching data: {e}")
+                st.error(f"❌ Error fetching data 2: {e}")
     elif view_mode == "Predictions":
         st.subheader("📊 Model Analysis")
         sub_option = st.radio("Select one of the following", ["India", "States", "Market"], horizontal=True)
@@ -1386,7 +1517,7 @@ if st.session_state.authenticated:
 
             if st.button("Forecast"):
                 query_filter = {"state": selected_state}
-                df = fetch_and_process_data(query_filter)
+                df = fetch_and_process_data(query_filter, collection)
                 if sub_timeline == "14 days":
                     forecast(df, filter_key, 14)
                 elif sub_timeline == "1 month":
@@ -1399,7 +1530,9 @@ if st.session_state.authenticated:
             filter_key = f"market_{selected_market}"  # Unique key for each market
             if st.button("Forecast"):
                 query_filter = {"Market Name": selected_market}
-                df = fetch_and_process_data(query_filter)
+                comparison_date = pd.to_datetime("18 Feb 2025")
+                df = fetch_and_process_data(query_filter, market_price_data)
+                st.write(df[df["Reported Date"]>comparison_date])
                 if sub_timeline == "14 days":
                     forecast(df, filter_key, 14)
                 elif sub_timeline == "1 month":
@@ -1412,7 +1545,7 @@ if st.session_state.authenticated:
             if True:
                 if st.button("Forecast"):
                     query_filter = {}
-                    df = fetch_and_process_data(query_filter)
+                    df = fetch_and_process_data(query_filter, collection)
                     if sub_timeline == "14 days":
                         forecast(df, "India", 14)
                     elif sub_timeline == "1 month":
@@ -1422,9 +1555,7 @@ if st.session_state.authenticated:
 
     elif view_mode=="Statistics":
         document = collection.find_one()
-        print(document)
         df = get_dataframe_from_collection(collection)
-        print(df)
         display_statistics(df)
     elif view_mode == "Exim":
         df = collection_to_dataframe(impExp)