remove end date

src/config/optimization_config.py

@@ -17,12 +17,39 @@ def get_date_span():
     try:
         # Try to get from streamlit session state (from config page)
         import streamlit as st
-        if hasattr(st, 'session_state') and 'start_date' in st.session_state and 'end_date' in st.session_state:
-            from datetime import datetime
+        if hasattr(st, 'session_state') and 'start_date' in st.session_state:
+            from datetime import datetime, timedelta
             start_date = datetime.combine(st.session_state.start_date, datetime.min.time())
-            end_date = datetime.combine(st.session_state.end_date, datetime.min.time())
-            date_span = list(range(1, (end_date - start_date).days + 2))
-            print(f"Using dates from config page: {start_date} to {end_date}")
+            
+            # Check if we have calculated planning_days, otherwise determine from data
+            if 'planning_days' in st.session_state and st.session_state.planning_days:
+                planning_days = st.session_state.planning_days
+                end_date = start_date + timedelta(days=planning_days - 1)
+            else:
+                # Determine date range from actual demand data for the exact start date
+                try:
+                    demand_data = extract.read_orders_data(start_date=start_date)
+                    if not demand_data.empty:
+                        import pandas as pd
+                        # Get unique finish dates for this exact start date
+                        finish_dates = pd.to_datetime(demand_data["Basic finish date"]).dt.date.unique()
+                        finish_dates = sorted(finish_dates)
+                        if finish_dates:
+                            end_date = datetime.combine(max(finish_dates), datetime.min.time())
+                            planning_days = (end_date - start_date).days + 1
+                        else:
+                            end_date = start_date
+                            planning_days = 1
+                    else:
+                        end_date = start_date + timedelta(days=4)  # Default 5 days
+                        planning_days = 5
+                except Exception as e:
+                    print(f"Could not determine date range from data: {e}")
+                    end_date = start_date + timedelta(days=4)  # Default 5 days
+                    planning_days = 5
+            
+            date_span = list(range(1, planning_days + 1))
+            print(f"Using dates from config page: {start_date} to {end_date} ({planning_days} days)")
             print("date span", date_span)
             return date_span, start_date, end_date
     except Exception as e:
@@ -37,10 +64,12 @@ def get_date_span():
 #fetch date from streamlit or default value. The streamlit and default references the demand data (COOIS_Planned_and_Released.csv)
 DATE_SPAN, start_date, end_date = get_date_span() 
 
+# Update global dates in extract module BEFORE any data loading
+extract.set_global_dates(start_date, end_date)
 
 print(f"\n📅 DATE RANGE: {start_date} to {end_date}")
 print(f"📁 PRODUCT SOURCE: COOIS_Released_Prod_Orders.csv")
-PRODUCT_LIST = transformed_data.get_released_product_list(start_date, end_date)
+PRODUCT_LIST = transformed_data.get_released_product_list(start_date)
 print(f"📦 PRODUCTS FOUND: {len(PRODUCT_LIST)} products -> {PRODUCT_LIST}")
 
 
@@ -212,7 +241,7 @@ def get_demand_dictionary():
     
     print(f"Loading default demand values from data files")
     # Use released orders instead of planned orders for demand
-    demand_df = extract.read_released_orders_data(start_date=start_date, end_date=end_date)
+    demand_df = extract.read_orders_data(start_date=start_date)
     demand_dictionary = demand_df.groupby('Material Number')["Order quantity (GMEIN)"].sum().to_dict()
     print(f"📈 DEMAND DATA: {len(demand_dictionary)} products with total demand {sum(demand_dictionary.values())}")
     return demand_dictionary
@@ -271,9 +300,11 @@ def get_team_requirements(PRODUCT_LIST):
         print(f"Using default value for team requirements, extracting from CSV: {e}")
         
     # Read the kits calculation data directly
-    kits_path = "data/real_data_excel/converted_csv/Kits__Calculation.csv"
-    kits_df = pd.read_csv(kits_path)
-    
+    kits_df = extract.read_personnel_requirement_data()
+    # kits_path = "data/real_data_excel/converted_csv/Kits__Calculation.csv"
+    # kits_df = pd.read_csv(kits_path)
+    print("kits_df columns:", kits_df.columns.tolist())
+    print("kits_df head:", kits_df.head())
     # Initialize the team requirements dictionary
     team_req_dict = {
         "UNICEF Fixed term": {},
@@ -282,17 +313,18 @@ def get_team_requirements(PRODUCT_LIST):
     
     # Process each product in the product list
     for product in PRODUCT_LIST:
+        print("product",product)
         print(f"Processing team requirements for product: {product}")
         product_data = kits_df[kits_df['Kit'] == product]
-        
+        print("product_data",product_data)
         if not product_data.empty:
             # Extract Humanizer and UNICEF staff requirements
             humanizer_req = product_data["Humanizer"].iloc[0]
             unicef_req = product_data["UNICEF staff"].iloc[0]
             
-            # Convert to int, handle NaN/empty values
-            team_req_dict["Humanizer"][product] = int(humanizer_req) if pd.notna(humanizer_req) else 0
-            team_req_dict["UNICEF Fixed term"][product] = int(unicef_req) if pd.notna(unicef_req) else 0
+            # Convert to int (data is already cleaned in extract function)
+            team_req_dict["Humanizer"][product] = int(humanizer_req)
+            team_req_dict["UNICEF Fixed term"][product] = int(unicef_req)
         else:
             print(f"Warning: Product {product} not found in Kits Calculation data, setting requirements to 0")
             

src/etl/extract.py

@@ -3,9 +3,17 @@ import datetime
 from datetime import date, timedelta
 import json
 import os
+# Default dates - will be overridden by optimization_config.py
 START_DATE = pd.Timestamp(2025, 7, 7)
 END_DATE = pd.Timestamp(2025, 7, 11)
 
+def set_global_dates(start_date, end_date):
+    """Update global START_DATE and END_DATE variables"""
+    global START_DATE, END_DATE
+    START_DATE = pd.Timestamp(start_date)
+    END_DATE = pd.Timestamp(end_date)
+    print(f"Updated global dates: {START_DATE} to {END_DATE}")
+
 
 def read_excel(path: str) -> pd.DataFrame:
     return pd.read_excel(path, dtype={"id": "Int64"})
@@ -13,13 +21,18 @@ def read_excel(path: str) -> pd.DataFrame:
 
 def read_demand_data(
     path="data/real_data_excel/converted_csv/COOIS_Planned_and_Released.csv",
+    start_date=None,
+    end_date=None,
 ) -> pd.DataFrame:
     df = pd.read_csv(path)
     df["Basic start date"] = pd.to_datetime(df["Basic start date"])
-    df["Basic finish date"] = pd.to_datetime(df["Basic finish date"])
-    df = df[
-        (df["Basic start date"] >= START_DATE) & (df["Basic finish date"] <= END_DATE)
-    ]
+    # df["Basic finish date"] = pd.to_datetime(df["Basic finish date"])
+    
+    # Use provided dates or fall back to module defaults
+    filter_start_date = start_date if start_date is not None else START_DATE
+    filter_end_date = end_date if end_date is not None else END_DATE
+    
+    df = df[(df["Basic start date"] == filter_start_date)]
 
     return df
 
@@ -67,10 +80,10 @@ def read_packaging_line_data(
     return df
 
 
-def read_released_orders_data(
-    path="data/real_data_excel/converted_csv/COOIS_Released_Prod_Orders.csv",
+def read_orders_data(
+    path="data/real_data_excel/converted_csv/COOIS_Planned_and_Released.csv",
     start_date=None,
-    end_date=None,
+    # end_date=None,
 ) -> pd.DataFrame:
     """
     COOIS_Released_Prod_Orders.csv 
@@ -78,7 +91,7 @@ def read_released_orders_data(
     Args:
         path: path to the csv file
         start_date: start date (pd.Timestamp or datetime)
-        end_date: end date (pd.Timestamp or datetime)
+    
 
     Returns:
         pd.DataFrame: filtered dataframe by date
@@ -87,15 +100,14 @@ def read_released_orders_data(
     assert len(df) > 0, "No data found in the file"
     # convert date column to datetime
     df["Basic start date"] = pd.to_datetime(df["Basic start date"])
-    df["Basic finish date"] = pd.to_datetime(df["Basic finish date"])
+    # df["Basic finish date"] = pd.to_datetime(df["Basic finish date"])
     
     
     # filter by date
-    if start_date is not None and end_date is not None:
-        # filter by date
-        df = df[
-            (df["Basic start date"] == pd.to_datetime(start_date)) & (df["Basic finish date"] == pd.to_datetime(end_date))
-        ]
+    if start_date is not None:    # Filter for exact start date only
+        df = df[df["Basic start date"] == pd.to_datetime(start_date)]
+    else:
+        raise ValueError("start_date is required")
     
     return df
 
@@ -104,10 +116,41 @@ def read_package_speed_data(
     path="data/real_data_excel/converted_csv/Kits__Calculation.csv",
 ):
     df = pd.read_csv(path, usecols=["Kit", "Kit per day","Paid work hours per day"])
+    df["Kit per day"] = df["Kit per day"].astype(float)
+    df["Paid work hours per day"] = df["Paid work hours per day"].astype(float)
+    df["Kit"] = df["Kit"].astype(str)
     df['kits_per_hour'] = df['Kit per day']/df['Paid work hours per day']
     speeds_per_hour = dict(zip(df["Kit"], df["kits_per_hour"]))
     return speeds_per_hour
 
+def read_personnel_requirement_data(
+    path="data/real_data_excel/converted_csv/Kits__Calculation.csv",
+):
+    df = pd.read_csv(path, usecols=["Kit", "Humanizer", "UNICEF staff"])
+    
+    # Clean the data by handling special whitespace characters like \xa0 (non-breaking space)
+    def clean_and_convert_to_float(value):
+        if pd.isna(value):
+            return 0.0
+        
+        # Convert to string and strip all kinds of whitespace (including \xa0)
+        clean_value = str(value).strip()
+        
+        # If empty after stripping, return 0
+        if clean_value == '' or clean_value == 'nan':
+            return 0.0
+        
+        try:
+            return float(clean_value)
+        except ValueError as e:
+            print(f"Warning: Could not convert '{repr(value)}' to float, setting to 0. Error: {e}")
+            return 0.0
+    
+    df["Humanizer"] = df["Humanizer"].apply(clean_and_convert_to_float)
+    df["UNICEF staff"] = df["UNICEF staff"].apply(clean_and_convert_to_float)
+    df["Kit"] = df["Kit"].astype(str)
+    
+    return df
 
 
 def get_production_order_data():