Update app.py

app.py

@@ -1145,6 +1145,7 @@ def fetch_and_store_data_market():
     latest_doc = collection.find_one(sort=[("Reported Date", -1)])
     if latest_doc and "Reported Date" in latest_doc:
         latest_date = latest_doc["Reported Date"]
+        st.write("Latest Date: ", latest_date)
     else:
         latest_date = None
 
@@ -1157,11 +1158,14 @@ def fetch_and_store_data_market():
     to_date = (datetime.now() - timedelta(days=1)).strftime('%d %b %Y')
     from_date_obj = datetime.strptime(from_date, '%d %b %Y')
     to_date_obj = datetime.strptime(to_date, '%d %b %Y')
-    if to_date_obj < from_date_obj:
-        print("Data already scraped")
+    st.write(to_date_obj)
+    st.write(from_date_obj)
+    
+    if to_date_obj <= from_date_obj:
+        st.write("Data already scraped")
         return None
-    # Build the URL to be requested
     base_url = "https://agmarknet.gov.in/SearchCmmMkt.aspx"
+    st.write(base_url)
     params = {
         "Tx_Commodity": "11",
         "Tx_State": "0",
@@ -1187,13 +1191,16 @@ def fetch_and_store_data_market():
     }
 
     response = requests.get(api_url, params=scraperapi_params)
-
+    st.write("Response: ", response.status_code)
     if response.status_code == 200:
         soup = BeautifulSoup(response.content, 'html.parser')
+        if soup:
+            st.write("Got html content")
+            st.write(soup.contents)
         table = soup.find("table", {"class": "tableagmark_new"})
-
         if table:
             headers = [th.get_text(strip=True) for th in table.find_all("th")]
+            st.write("Got all headers")
             rows = []
 
             for row in table.find_all("tr")[1:]:
@@ -1216,9 +1223,11 @@ def fetch_and_store_data_market():
             else:
                 print("No new records to insert.")
             return df
+        else:
+            st.write("No table found")
 
     else:
-        print(f"Failed to fetch data with status code: {response.status_code}")
+        st.write(f"Failed to fetch data with status code: {response.status_code}")
         return None
 
 
@@ -1316,8 +1325,8 @@ if st.session_state.authenticated:
     st.title("🌾 AgriPredict Dashboard")
     if st.button("Get Live Data Feed"):
         fetch_and_store_data()
+        st.write("Now executing Market scraper: ")
         fetch_and_store_data_market()
-    # Top-level radio buttons for switching views
     view_mode = st.radio("", ["Statistics", "Plots", "Predictions", "Exim"], horizontal=True)
 
     if view_mode == "Plots":