Finalize ETL and optimization

data/real_data_excel/converted_csv/WH_Workforce_Hourly_Pay_Scale_processed.csv

@@ -5,7 +5,7 @@ id,employment_type_id,employment_type,shift,shift_type_id,level,Value (USD)
 3,1,UNICEF Fixed term,Day shift,1, GS4 ,43.27451923076923
 3,1,UNICEF Fixed term,evening_shift,2, GS4 ,43.27451923076923
 4,1,UNICEF Fixed term,overtime,3, GS4 ,64.91177884615385
-5,1,UNICEF Fixed term,Day shift,1, GS5 ,44.35817307692308,
+5,1,UNICEF Fixed term,Day shift,1, GS5 ,44.35817307692308
 5,1,UNICEF Fixed term,evening_shift,2, GS5 ,44.35817307692308
 6,1,UNICEF Fixed term,overtime,3, GS5 ,66.53725961538461
 7,1,UNICEF Fixed term,Day shift,1, GS7 ,57.666826923076925

data/real_data_excel/converted_csv/Work_Centre_Capacity_processed.csv

@@ -5,4 +5,4 @@ id,Work_Area,per_hour_capacity,per_hour_unit,line_for_packaging,line_count
 4,Pre-pack_lines,300.0,cartons,False,1
 5,Pre-pick_station,54.0,pallets,False,1
 6,Long_line,300.0,cartons,True,2
-7,Short_ine,54.0,pallets,True,2
+7,Short_ine,54.0,pallets,True,2

src/config/optimization_config.py

@@ -10,12 +10,15 @@ def get_date_span():
     try: 
         start_date = dashboard.start_date
         end_date = dashboard.end_date
-        date_span = list(range(1, (end_date - start_date).days + 1))
+        date_span = list(range(1, (end_date - start_date).days + 2))
         print(f"date from user input")
+        print("date span",date_span)
+        print("start date",start_date)
+        print("end date",end_date)
         return date_span, start_date, end_date
     except Exception as e:
         print(f"using default value for date span")
-        return list(range(1, 5)), datetime(2025, 3, 24), datetime(2025, 3, 28)  # Default 7 days
+        return list(range(1, 6)), datetime(2025, 3, 24), datetime(2025, 3, 29)  # Default 7 days
 
 
 #fetch date from streamlit or default value. The streamlit and default references the demand data (COOIS_Planned_and_Released.csv)
@@ -28,17 +31,17 @@ print("product list",PRODUCT_LIST)
 
 def get_employee_type_list():
     try:
-        pass
-        # streamlit_employee_type_list = dashboard.employee_type_list
-        # return streamlit_employee_type_list
-    except Exception as e:
-        print(f"using default value for employee type list")
+        streamlit_employee_type_list = dashboard.employee_type_list
+        return streamlit_employee_type_list
+    except (AttributeError, Exception) as e:
+        print(f"using default value for employee type list: {e}")
         employee_type_list = extract.read_employee_data()
+        # print("employee type df",employee_type_list)
         emp_type_list = employee_type_list["employment_type"].unique()
         return emp_type_list
-
+        
 EMPLOYEE_TYPE_LIST = get_employee_type_list()
-print("employee type list",EMPLOYEE_TYPE_LIST)
+# print("employee type list",EMPLOYEE_TYPE_LIST)
 
 def get_shift_list():
     try:
@@ -46,11 +49,11 @@ def get_shift_list():
         return streamlit_shift_list
     except Exception as e:
         print(f"using default value for shift list")
-        shift_list = extract.read_shift_cost_data()
+        shift_list = extract.get_shift_info()
         shift_list = shift_list["id"].unique()
         return shift_list
 SHIFT_LIST = get_shift_list()
-print(SHIFT_LIST)
+# print("shift list",SHIFT_LIST)
 
 
 def get_line_list():
@@ -64,7 +67,7 @@ def get_line_list():
         return line_list
 
 LINE_LIST = get_line_list()
-print(LINE_LIST)
+# print("line list",LINE_LIST)
 
 def get_line_cnt_per_type():
     try:
@@ -74,11 +77,12 @@ def get_line_cnt_per_type():
         print(f"using default value for line cnt per type")
         line_df = extract.read_packaging_line_data()
         line_cnt_per_type = line_df.set_index("id")["line_count"].to_dict()
+        print("line cnt per type")
         print(line_cnt_per_type)
         return line_cnt_per_type
 
 LINE_CNT_PER_TYPE = get_line_cnt_per_type()
-print(LINE_CNT_PER_TYPE)
+print("line cnt per type",LINE_CNT_PER_TYPE)
 
 def get_demand_dictionary():
     try:
@@ -86,12 +90,13 @@ def get_demand_dictionary():
         return streamlit_demand_dictionary
     except Exception as e:
         print(f"using default value for demand dictionary")
-        demand_df = extract.read_demand_data()
+        # Use released orders instead of planned orders for demand
+        demand_df = extract.read_released_orders_data(start_date=start_date, end_date=end_date)
         demand_dictionary = demand_df.groupby('Material Number')["Order quantity (GMEIN)"].sum().to_dict()
         return demand_dictionary
 
 DEMAND_DICTIONARY = get_demand_dictionary()
-print(DEMAND_DICTIONARY)
+# print("demand dictionary",DEMAND_DICTIONARY)
 
 def get_cost_list_per_emp_shift():
     try:
@@ -102,10 +107,10 @@ def get_cost_list_per_emp_shift():
         shift_cost_df = extract.read_shift_cost_data()
         #question - Important : there is multiple type of employment type in terms of the cost
         #1 -. unicef 2 - humanizer
-        return {1:{1:43,2:43,6:64},2:{1:27,2:27,6:41}}
+        return {"UNICEF Fixed term":{1:43,2:43,3:64},"Humanizer":{1:27,2:27,3:41}}
 
 COST_LIST_PER_EMP_SHIFT = get_cost_list_per_emp_shift()
-print(COST_LIST_PER_EMP_SHIFT)
+# print("cost list per emp shift",COST_LIST_PER_EMP_SHIFT)
         
 
 
@@ -156,7 +161,7 @@ def get_productivity(PRODUCT_LIST):
         return productivity_list_per_emp_product
 
 PRODUCTIVITY_LIST_PER_EMP_PRODUCT = get_productivity(PRODUCT_LIST)
-print(PRODUCTIVITY_LIST_PER_EMP_PRODUCT)
+# print("productivity list per emp product",PRODUCTIVITY_LIST_PER_EMP_PRODUCT)
 
 
 # PRODUCTIVITY_LIST_PER_EMP_PRODUCT = {  # Kits_Calculation
@@ -184,10 +189,19 @@ MAX_EMPLOYEE_PER_TYPE_ON_DAY = {  # Not available information
 MAX_HOUR_PER_PERSON_PER_DAY = 14  # legal standard
 MAX_HOUR_PER_SHIFT_PER_PERSON = {1: 7, 2: 7, 3: 3}  # work_shifts_timing.csv #Need to be fixed
 CAP_PER_LINE_PER_HOUR = {
-    "long": 2200,
-    "short": 1600,
+    6: 2200, #long
+    7: 1600, #short
 }
 # number of products that can be produced per hour per line
 #This information is critical and it should not rely on the productivity information
 
 DAILY_WEEKLY_SCHEDULE = "daily"  # daily or weekly ,this needs to be implementedin  in         if F_x1_day is not None... F_x1_week is not None... also need to change x1 to Fixedstaff_first_shift
+
+# Fixed staff constraint mode
+# Options:
+#   "mandatory" - Forces all fixed staff to work full hours every day (expensive, 99.7% waste)
+#   "available" - Staff available up to limits but not forced (balanced approach)
+#   "priority"  - Fixed staff used first, then temporary staff (realistic business model)
+#   "none"      - Purely demand-driven scheduling (cost-efficient)
+FIXED_STAFF_CONSTRAINT_MODE = "priority"  # Recommended: "priority" for realistic business model
+

src/etl/extract.py

@@ -28,9 +28,15 @@ def read_employee_data(
 ) -> pd.DataFrame:
     return pd.read_csv(path)
 
+def get_shift_info(
+        path = "data/real_data_excel/converted_csv/work_shift.csv"
+) -> pd.DataFrame:
+    df = pd.read_csv(path)
+    return df
+
 
 def read_shift_cost_data(
-    path="data/real_data_excel/converted_csv/work_shift.csv",
+    path="data/real_data_excel/converted_csv/WH_Workforce_Hourly_Pay_Scale_processed.csv",
 ) -> pd.DataFrame:
     return pd.read_csv(path)
 
@@ -95,5 +101,7 @@ def read_released_orders_data(
 
 
 if __name__ == "__main__":
-    demand_data = read_demand_data()
-    print(demand_data.head())
+    employee_data = read_employee_data()
+    print("employee data")
+    print(employee_data)
+    

src/etl/models.py

@@ -1 +0,0 @@
-

src/models/optimizer_real.py

@@ -48,6 +48,13 @@ class OptimizerReal:
         # -----------------------------
         # Weekly demand (units) for each product in product_list
         weekly_demand = self.config.DEMAND_DICTIONARY
+        
+        # Validate demand - check if any products have positive demand
+        total_demand = sum(weekly_demand.get(p, 0) for p in product_list)
+        if total_demand == 0:
+            print("Warning: Total demand is zero for all products. Optimization may not be meaningful.")
+            print("Products:", product_list)
+            print("Demands:", {p: weekly_demand.get(p, 0) for p in product_list})
 
         # Daily activity toggle for each product (1=can be produced on day t; 0=cannot)
         # If a product is not active on a day, we force its production and hours to 0 that day.
@@ -76,17 +83,33 @@ class OptimizerReal:
         Cap = self.config.CAP_PER_LINE_PER_HOUR
 
         # Fixed regular hours for type Fixed on shift 1
-        # Choose either PER-DAY values or a single PER-WEEK total.
-        # Common in practice: per-day fixed hours (regulars show up daily).
-        # F_x1_day = Fixed working hour for fixed staff on shift 1
+        # BUSINESS LOGIC: Fixed staff availability vs mandatory hours
+        # Controlled by FIXED_STAFF_CONSTRAINT_MODE in optimization_config
+        
         first_shift_hour = Hmax_shift[1]
         daily_weekly_type = self.config.DAILY_WEEKLY_SCHEDULE
-        print(first_shift_hour)
-        F_x1_day = {
-            t: first_shift_hour * N_day["Fixed"][t] + 1 for t in days
-        }  # EDIT if different from "all regulars do full usual shift"
-        print(F_x1_day)
-        F_x1_week = None  # e.g., sum(F_x1_day.values()) if you want weekly instead (then set F_x1_day=None)
+        constraint_mode = self.config.FIXED_STAFF_CONSTRAINT_MODE
+        
+        if constraint_mode == "mandatory":
+            # Option 1: Mandatory hours (forces staff to work even when idle)
+            F_x1_day = {t: first_shift_hour * N_day["UNICEF Fixed term"][t] for t in days}
+            print(f"Using MANDATORY fixed hours constraint: {sum(F_x1_day.values())} hours/week")
+        elif constraint_mode == "available":
+            # Option 2: Available hours constraint (staff can work up to limit but not forced)
+            F_x1_day = None
+            print("Using AVAILABLE hours constraint (not yet implemented)")
+        elif constraint_mode == "priority":
+            # Option 3: Priority-based (realistic business model)
+            F_x1_day = None
+            print("Using PRIORITY constraint - fixed staff first, then temporary staff")
+        elif constraint_mode == "none":
+            # Option 4: No constraint (fully demand-driven)
+            F_x1_day = None
+            print("Using NO fixed hours constraint - demand-driven scheduling")
+        else:
+            raise ValueError(f"Invalid FIXED_STAFF_CONSTRAINT_MODE: {constraint_mode}. Use 'mandatory', 'available', 'priority', or 'none'")
+        
+         # e.g., F_x1_day =  sum(F_x1_day.values()) if you want weekly instead (then set F_x1_day=None)
         cap_per_line_per_hour = self.config.CAP_PER_LINE_PER_HOUR
         # Optional skill/compatibility: allow[(e,p,ell)] = 1/0 (1=allowed; 0=forbid)
         allow = {}
@@ -107,6 +130,7 @@ class OptimizerReal:
         # 4) DECISION VARIABLES
         # -----------------------------
         # h[e,s,p,ell,t] = worker-hours of type e on shift s for product p on line ell on day t (integer)
+
         h = {}
         for e in employee_types:
             for s in shift_list:
@@ -114,6 +138,7 @@ class OptimizerReal:
                     for ell in line_type_cnt_tuple:
                         for t in days:
                             # Upper bound per (e,s,t): shift cap * available headcount that day
+                            
                             ub = Hmax_shift[s] * N_day[e][t]
                             h[e, s, p, ell, t] = solver.IntVar(
                                 0, ub, f"h_{e}_{s}_{p}_{ell[0]}{ell[1]}_d{t}"
@@ -148,6 +173,8 @@ class OptimizerReal:
         # -----------------------------
         # 5) OBJECTIVE: Minimize total labor cost over the week
         # -----------------------------
+        print("wage_types",wage_types)
+        print("h",h)
         solver.Minimize(
             solver.Sum(
                 wage_types[e][s] * h[e, s, p, ell, t]
@@ -165,10 +192,12 @@ class OptimizerReal:
 
         # 6.1 Weekly demand (no daily demand)
         for p in product_list:
-            solver.Add(
-                solver.Sum(u[p, ell, s, t] for ell in line_type_cnt_tuple for s in shift_list for t in days)
-                >= weekly_demand.get(p, 0)
-            )
+            demand_value = weekly_demand.get(p, 0)
+            if demand_value > 0:  # Only add constraint if there's actual demand
+                solver.Add(
+                    solver.Sum(u[p, ell, s, t] for ell in line_type_cnt_tuple for s in shift_list for t in days)
+                    >= demand_value
+                )
 
         # 6.2 If a product is inactive on a day, force zero production and hours for that day
         # This makes "varying products per day" explicit.
@@ -195,6 +224,9 @@ class OptimizerReal:
                         )
 
         # 6.4 Per-line throughput cap (units/hour × line-hours)
+        print("================================================")
+        print("cap_per_line_per_hour",cap_per_line_per_hour)
+        print("tline",tline)
         for ell in line_type_cnt_tuple:
             for s in shift_list:
                 for t in days:
@@ -223,24 +255,55 @@ class OptimizerReal:
 
         # 6.6 Fixed regular hours for type Fixed on shift 1
         if F_x1_day is not None:
-            # Per-day fixed hours
+            # Per-day fixed hours (mandatory - expensive)
             for t in days:
                 solver.Add(
-                    solver.Sum(h["Fixed", 1, p, ell, t] for p in product_list for ell in line_type_cnt_tuple)
+                    solver.Sum(h["UNICEF Fixed term", 1, p, ell, t] for p in product_list for ell in line_type_cnt_tuple)
                     == F_x1_day[t]
                 )
-        elif F_x1_week is not None:
-            # Per-week fixed hours
-            solver.Add(
-                solver.Sum(
-                    h["Fixed", 1, p, ell, t] for p in product_list for ell in line_type_cnt_tuple for t in days
-                )
-                == F_x1_week
-            )
+            print("Applied mandatory fixed hours constraint")
+        # elif F_x1_week is not None:
+        #     # Per-week fixed hours (mandatory - expensive)
+        #     solver.Add(
+        #         solver.Sum(
+        #             h["UNICEF Fixed term", 1, p, ell, t] for p in product_list for ell in line_type_cnt_tuple for t in days
+        #         )
+        #         == F_x1_week
+        #     )
+        #     print("Applied mandatory weekly fixed hours constraint")
         else:
-            raise ValueError(
-                "Specify either F_x1_day (dict by day) or F_x1_week (scalar)."
-            )
+            # No fixed constraint - purely demand-driven (cost-efficient)
+            print("No mandatory fixed hours constraint - using demand-driven scheduling")
+            # The availability constraint (6.7) already limits maximum hours
+            
+            # Special handling for priority mode
+            if constraint_mode == "priority":
+                print("Implementing priority constraints: UNICEF Fixed term used before Humanizer")
+                # Add constraints to prioritize fixed staff usage before temporary staff
+                
+                # Priority constraint: For each day, product, and line,
+                # Humanizer hours can only be used if UNICEF Fixed term is at capacity
+                for t in days:
+                    for p in product_list:
+                        for ell in line_type_cnt_tuple:
+                            # Create binary variable to indicate if UNICEF Fixed term is at capacity
+                            unicef_at_capacity = solver.IntVar(0, 1, f"unicef_at_capacity_{p}_{ell[0]}{ell[1]}_d{t}")
+                            
+                            # UNICEF Fixed term capacity for this specific (p, ell, t)
+                            max_unicef_hours = Hmax_shift[1] * N_day["UNICEF Fixed term"][t]
+                            
+                            # If UNICEF is not at capacity (unicef_at_capacity = 0), then Humanizer hours must be 0
+                            # If UNICEF is at capacity (unicef_at_capacity = 1), then Humanizer can work
+                            solver.Add(
+                                solver.Sum(h["Humanizer", s, p, ell, t] for s in shift_list) 
+                                <= unicef_at_capacity * max_unicef_hours  # Large M value
+                            )
+                            
+                            # Force unicef_at_capacity = 1 only when UNICEF hours are significant
+                            solver.Add(
+                                solver.Sum(h["UNICEF Fixed term", s, p, ell, t] for s in shift_list)
+                                >= unicef_at_capacity * 0.1  # Small threshold to trigger capacity flag
+                            )
 
         # 6.7 Daily hours cap per employee type (14h per person per day)
         for e in employee_types: