demand filtering / visualization done.

app.py

@@ -45,12 +45,15 @@ elif page == "📊 Optimization Results":
     
     # Check if we have results in session state
     if 'optimization_results' in st.session_state and st.session_state.optimization_results:
+        
         display_optimization_results(st.session_state.optimization_results)
+        
     else:
         st.title("📊 Optimization Results")
         st.info("🔄 No optimization results available yet.")
         st.markdown("Please run an optimization from the **⚙️ Settings** page first to see results here.")
         
+        
         # Add helpful instructions
         st.markdown("### 📋 How to Get Results:")
         st.markdown("1. Go to **⚙️ Settings** page")

config_page.py

@@ -18,6 +18,8 @@ def render_config_page():
     
     st.title("⚙️ Settings")
     st.markdown("---")
+    
+
     st.markdown("Adjust the settings for your workforce optimization. These settings control how the system schedules employees and calculates costs.")
     
     # Initialize session state for all configuration values
@@ -48,8 +50,28 @@ def render_config_page():
         if st.button("💾 Save Settings", type="primary", use_container_width=True):
             config = save_configuration()
             st.success("✅ Settings saved successfully!")
-            # Trigger demand validation after saving settings
+            
+            # Clear any previous optimization results since settings changed
+            if 'optimization_results' in st.session_state:
+                del st.session_state.optimization_results
+            
+            # Clear any previous validation state
+            if 'show_validation_after_save' in st.session_state:
+                del st.session_state.show_validation_after_save
+            if 'settings_just_saved' in st.session_state:
+                del st.session_state.settings_just_saved
+            
+            # Clear any cached validation results since settings changed
+            validation_cache_key = f"validation_results_{st.session_state.get('start_date', 'default')}"
+            if validation_cache_key in st.session_state:
+                del st.session_state[validation_cache_key]
+            
+            # Trigger fresh demand validation after saving settings
             st.session_state.show_validation_after_save = True
+            st.session_state.settings_just_saved = True
+            
+            # Force a page refresh to show the updated validation
+            st.rerun()
     
     # Display settings summary at full width (outside columns)
     st.markdown("---")
@@ -58,27 +80,133 @@ def render_config_page():
             display_user_friendly_summary(st.session_state.optimization_config)
     
     # Show demand validation after saving settings
-    if st.session_state.get('show_validation_after_save', False):
+    if st.session_state.get('show_validation_after_save', False) or st.session_state.get('settings_just_saved', False):
         st.markdown("---")
         st.header("📋 Data Validation Results")
         st.markdown("Analyzing your demand data to identify potential optimization issues...")
         
-        try:
-            from src.demand_validation_viz import display_demand_validation
-            display_demand_validation()
+        # Check if we have cached validation results
+        validation_cache_key = f"validation_results_{st.session_state.get('start_date', 'default')}"
+        
+        if validation_cache_key not in st.session_state:
+            # Run validation and cache results
+            with st.spinner("🔄 Running data validation..."):
+                try:
+                    from src.demand_validation_viz import DemandValidationViz
+                    
+                    # Initialize validator and run validation
+                    validator = DemandValidationViz()
+                    print("validator",validator)
+                    if validator.load_data():
+                        
+                        validation_df = validator.validate_all_products()
+                        print("validation_df", validation_df)
+                        summary_stats = validator.get_summary_statistics(validation_df)
+                        print("summary_stats", summary_stats)
+                        # Cache the results
+                        st.session_state[validation_cache_key] = {
+                            'validation_df': validation_df,
+                            'summary_stats': summary_stats,
+                            'validator': validator
+                        }
+                    else:
+                        st.error("❌ Failed to load validation data")
+                        st.session_state[validation_cache_key] = None
+                        
+                except Exception as e:
+                    st.error(f"❌ Error in demand validation: {str(e)}")
+                    st.session_state[validation_cache_key] = None
+        
+        # Display cached validation results
+        if st.session_state.get(validation_cache_key):
+            # print("cached_results",st.session_state[validation_cache_key])
+            cached_results = st.session_state[validation_cache_key]
+            
+            validation_df = cached_results['validation_df']
+            # print("validation_df",validation_df)
+            summary_stats = cached_results['summary_stats']
+            # print("summary_stats",summary_stats)
+            # Display summary statistics
+            st.subheader("📊 Summary Statistics")
+            
+            col1, col2, col3, col4 = st.columns(4)
+            
+            with col1:
+                st.metric("Total Products", summary_stats['total_products'])
+                st.metric("Included in Optimization", summary_stats['included_products'], delta="Ready for optimization")
+            
+            with col2:
+                st.metric("Total Demand", f"{summary_stats['total_demand']:,}")
+                st.metric("Excluded from Optimization", summary_stats['excluded_products'], delta="Omitted")
+            
+            with col3:
+                st.metric("Included Demand", f"{summary_stats['included_demand']:,}", delta="Will be optimized")
+                st.metric("UNICEF Staff Needed", summary_stats['total_unicef_needed'])
+            
+            with col4:
+                st.metric("Excluded Demand", f"{summary_stats['excluded_demand']:,}", delta="Omitted")
+                st.metric("Humanizer Staff Needed", summary_stats['total_humanizer_needed'])
+            
+            # Separate the results into included and excluded
+            included_df = validation_df[validation_df['Excluded from Optimization'] == False].copy()
+            excluded_df = validation_df[validation_df['Excluded from Optimization'] == True].copy()
+            
+            # Products Included in Optimization
+            st.subheader("✅ Products Included in Optimization")
+            st.write(f"**{len(included_df)} products** will be included in the optimization with total demand of **{included_df['Demand'].sum():,} units**")
+            
+            if len(included_df) > 0:
+                # Configure column display for included
+                included_columns = ['Product ID', 'Demand', 'Product Type', 'Line Type', 'UNICEF Staff', 'Humanizer Staff', 'Production Speed (units/hour)', 'Validation Status']
+                
+                st.dataframe(
+                    included_df[included_columns],
+                    use_container_width=True,
+                    height=300
+                )
+            else:
+                st.warning("No products are included in optimization!")
+            
+            # Products Excluded from Optimization  
+            st.subheader("🚫 Products Excluded from Optimization")
+            st.write(f"**{len(excluded_df)} products** are excluded from optimization with total demand of **{excluded_df['Demand'].sum():,} units**")
+            st.info("These products are omitted from optimization due to missing line assignments or zero staffing requirements.")
+            
+            if len(excluded_df) > 0:
+                # Show exclusion breakdown
+                exclusion_reasons = excluded_df['Exclusion Reasons'].value_counts()
+                st.write("**Exclusion reasons:**")
+                for reason, count in exclusion_reasons.items():
+                    st.write(f"• {reason}: {count} products")
+                
+                # Configure column display for excluded
+                excluded_columns = ['Product ID', 'Demand', 'Product Type', 'Exclusion Reasons', 'UNICEF Staff', 'Humanizer Staff', 'Line Type']
+                
+                st.dataframe(
+                    excluded_df[excluded_columns],
+                    use_container_width=True,
+                    height=200
+                )
+            else:
+                st.info("No products are excluded from optimization.")
             
             # Show validation reminder before optimization
             st.info("💡 **Review validation results above before running optimization.** " +
                    "Fix any critical issues (especially missing line assignments) to improve optimization success.")
-            
-        except Exception as e:
-            st.error(f"❌ Error in demand validation: {str(e)}")
+        else:
+            st.error("❌ Validation failed to run properly")
             st.info("💡 You can still proceed with optimization, but data issues may cause problems.")
         
-        # Reset the flag so validation doesn't show every time
-        if st.button("✅ Validation Reviewed - Continue to Optimization"):
-            st.session_state.show_validation_after_save = False
-            st.rerun()
+        # Reset the flags so validation doesn't show every time
+        col1, col2, col3 = st.columns([1, 1, 1])
+        with col2:
+            if st.button("✅ Validation Reviewed - Continue to Optimization", use_container_width=True):
+                st.session_state.show_validation_after_save = False
+                st.session_state.settings_just_saved = False
+                # Clear validation cache to force fresh validation next time
+                if validation_cache_key in st.session_state:
+                    del st.session_state[validation_cache_key]
+                st.rerun()  # Refresh to hide validation section
     
     # Optimization section
     st.markdown("---")
@@ -109,6 +237,8 @@ def render_config_page():
         # Show status of current results
         if 'optimization_results' in st.session_state and st.session_state.optimization_results is not None:
             st.success("✅ Results Available")
+        elif 'optimization_config' in st.session_state:
+            st.info("🔄 Settings Saved - Ready to Optimize")
         else:
             st.info("⏳ No Results Yet")
     
@@ -293,7 +423,7 @@ def render_workforce_config():
         max_value=20,
         value=st.session_state.fixed_min_unicef_per_day,
         help="Minimum number of UNICEF Fixed term employees required every working day (constraint)"
-    )
+        )
     
     # Working hours configuration
     st.subheader("⏰ Working Hours Configuration")
@@ -582,17 +712,21 @@ def save_configuration():
                 calculated_days = (calculated_end_date - st.session_state.start_date).days + 1
                 st.info(f"📊 Found demand data starting on {st.session_state.start_date} ending on {calculated_end_date} ({calculated_days} days, {len(demand_data)} orders)")
             else:
-                calculated_end_date = st.session_state.start_date
-                calculated_days = 1
+                raise Exception("No finish date found for start date")
+                # calculated_end_date = st.session_state.start_date
+                # calculated_days = 1
         else:
-            calculated_end_date = st.session_state.start_date
-            calculated_days = 1
+            # calculated_end_date = st.session_state.start_date
+            # calculated_days = 1
             st.warning(f"⚠️ No demand data found for start date {st.session_state.start_date}")
+            raise Exception("No demand data found for start date")
     except Exception as e:
         st.warning(f"Could not determine date range from data: {e}. Using default 5-day period.")
-        from datetime import timedelta
-        calculated_end_date = st.session_state.start_date + timedelta(days=4)
-        calculated_days = 5
+        raise Exception("Could not determine date range from data")
+        
+        # from datetime import timedelta
+        # calculated_end_date = st.session_state.start_date + timedelta(days=4)
+        # calculated_days = 5
     
     # Store calculated values in session state for compatibility
     st.session_state.end_date = calculated_end_date
@@ -602,8 +736,8 @@ def save_configuration():
     config = {
         'date_range': {
             'start_date': st.session_state.start_date,
-            'end_date': calculated_end_date,
-            'planning_days': calculated_days,
+            'end_date': st.session_state.end_date,
+            'planning_days': st.session_state.planning_days,
         },
         'schedule_type': st.session_state.schedule_type,
         'evening_shift_mode': st.session_state.evening_shift_mode,
@@ -657,7 +791,7 @@ def save_configuration():
     }
     
     # Calculate date span for proper employee limits (use calculated planning_days)
-    date_span_length = calculated_days
+    date_span_length = st.session_state.planning_days
     date_span = list(range(1, date_span_length + 1))
     
     # Store individual items in session state for optimization_config.py to access
@@ -931,6 +1065,7 @@ def run_optimization():
     """Run the optimization model and store results"""
     try:
         st.info("🔄 Running optimization... This may take a few moments.")
+    
         
         # Always clear everything first for clean slate
         clear_all_cache_and_results()
@@ -1018,9 +1153,9 @@ def check_critical_data_issues():
             warnings.append(f"{summary_stats['no_speed']} products missing production speed data")
         
         # Calculate failure risk
-        invalid_ratio = summary_stats['invalid_products'] / summary_stats['total_products']
-        if invalid_ratio > 0.5:
-            warnings.append(f"High failure risk: {invalid_ratio:.0%} of products have data issues")
+        # invalid_ratio = summary_stats['invalid_products'] / summary_stats['total_products']
+        # if invalid_ratio > 0.5:
+        #     warnings.append(f"High failure risk: {invalid_ratio:.0%} of products have data issues")
         
         return warnings
         

optimization_results.py

@@ -94,7 +94,8 @@ def display_weekly_summary(results):
     with col3:
         # Calculate fulfillment rate
         sys.path.append('src')
-        from config.optimization_config import DEMAND_DICTIONARY
+        from config.optimization_config import get_demand_dictionary
+        DEMAND_DICTIONARY = get_demand_dictionary()
         total_demand = sum(DEMAND_DICTIONARY.values())
         fulfillment_rate = (total_production / total_demand * 100) if total_demand > 0 else 0
         st.metric("Fulfillment Rate", f"{fulfillment_rate:.1f}%")
@@ -109,7 +110,8 @@ def display_weekly_summary(results):
     # Production vs Demand Chart
     st.subheader("🎯 Production vs Demand")
     
-    from config.optimization_config import DEMAND_DICTIONARY
+    from config.optimization_config import get_demand_dictionary
+    DEMAND_DICTIONARY = get_demand_dictionary()
     prod_demand_data = []
     for product, production in results['weekly_production'].items():
         demand = DEMAND_DICTIONARY.get(product, 0)
@@ -157,7 +159,8 @@ def display_daily_deep_dive(results):
         shift_name = {1: 'Regular', 2: 'Evening', 3: 'Overtime'}.get(row['shift'], f"Shift {row['shift']}")
         
         # Get team requirements for this production run
-        from src.config.optimization_config import TEAM_REQ_PER_PRODUCT
+        from src.config.optimization_config import get_team_requirements
+        TEAM_REQ_PER_PRODUCT = get_team_requirements()
         
         for emp_type in ['UNICEF Fixed term', 'Humanizer']:
             if row['product'] in TEAM_REQ_PER_PRODUCT.get(emp_type, {}):
@@ -263,7 +266,9 @@ def display_line_schedules(results):
     # Process schedule data
     schedule_data = []
     sys.path.append('src')
-    from config.optimization_config import TEAM_REQ_PER_PRODUCT, shift_code_to_name, line_code_to_name, DEMAND_DICTIONARY
+    from config.optimization_config import get_team_requirements, get_demand_dictionary, shift_code_to_name, line_code_to_name
+    TEAM_REQ_PER_PRODUCT = get_team_requirements()
+    DEMAND_DICTIONARY = get_demand_dictionary()
     
     # Get the mapping dictionaries
     shift_names = shift_code_to_name()
@@ -501,7 +506,8 @@ def display_kit_production(results):
     # Weekly production summary
     production_data = []
     sys.path.append('src')
-    from config.optimization_config import DEMAND_DICTIONARY
+    from config.optimization_config import get_demand_dictionary
+    DEMAND_DICTIONARY = get_demand_dictionary()
     
     for product, production in results['weekly_production'].items():
         demand = DEMAND_DICTIONARY.get(product, 0)
@@ -536,7 +542,8 @@ def display_cost_analysis(results):
     
     # Calculate cost breakdown
     sys.path.append('src')
-    from config.optimization_config import COST_LIST_PER_EMP_SHIFT, TEAM_REQ_PER_PRODUCT, shift_code_to_name, line_code_to_name
+    from config.optimization_config import COST_LIST_PER_EMP_SHIFT, get_team_requirements, shift_code_to_name, line_code_to_name
+    TEAM_REQ_PER_PRODUCT = get_team_requirements()
     
     # Get the mapping dictionaries
     shift_names = shift_code_to_name()
@@ -717,7 +724,8 @@ def display_input_data_inspection():
                     shift_name = ShiftType.get_name(shift_id)
                     st.write(f"• {shift_name}: {hours} hours/shift")
                 
-                max_daily_hours = optimization_config.get_max_hour_per_person_per_day()
+                from src.config.optimization_config import MAX_HOUR_PER_PERSON_PER_DAY
+                max_daily_hours = MAX_HOUR_PER_PERSON_PER_DAY
                 st.write(f"• Maximum daily hours per person: {max_daily_hours}")
         
         with st.expander("👥 **Workforce Parameters**", expanded=False):
@@ -774,7 +782,8 @@ def display_input_data_inspection():
             
             with col2:
                 st.write("**Product-Line Matching:**")
-                kit_line_match = optimization_config.get_kit_line_match_dict()
+                from src.config.optimization_config import KIT_LINE_MATCH_DICT
+                kit_line_match = KIT_LINE_MATCH_DICT
                 st.write("*Sample mappings:*")
                 sample_items = list(kit_line_match.items())[:10]
                 for product, line_type in sample_items:
@@ -912,11 +921,13 @@ def display_input_data_inspection():
             
             with col1:
                 st.write("**Schedule Mode:**")
-                schedule_mode = optimization_config.get_daily_weekly_schedule()
+                from src.config.optimization_config import DAILY_WEEKLY_SCHEDULE
+                schedule_mode = DAILY_WEEKLY_SCHEDULE
                 st.write(f"• Planning mode: {schedule_mode}")
                 
                 st.write("**Evening Shift Mode:**")
-                evening_mode = optimization_config.get_evening_shift_mode()
+                from src.config.optimization_config import EVENING_SHIFT_MODE
+                evening_mode = EVENING_SHIFT_MODE
                 evening_threshold = optimization_config.get_evening_shift_demand_threshold()
                 st.write(f"• Mode: {evening_mode}")
                 st.write(f"• Activation threshold: {evening_threshold:.1%}")

src/config/optimization_config.py

@@ -70,16 +70,32 @@ extract.set_global_dates(start_date, end_date)
 print(f"\n📅 DATE RANGE: {start_date} to {end_date}")
 print(f"📁 PRODUCT SOURCE: COOIS_Released_Prod_Orders.csv")
 
-# Get filtered products ready for optimization
-try:
-    from src.demand_filtering import get_filtered_product_list
-    PRODUCT_LIST = get_filtered_product_list()
-    print(f"📦 FILTERED PRODUCTS: {len(PRODUCT_LIST)} products ready for optimization")
-    print(f"🎯 Products: {PRODUCT_LIST}")
-except ImportError:
-    # Fallback to unfiltered list if demand_filtering not available
-    PRODUCT_LIST = transformed_data.get_released_product_list(start_date)
-    print(f"📦 UNFILTERED PRODUCTS: {len(PRODUCT_LIST)} products -> {PRODUCT_LIST}")
+def get_product_list():
+    """
+    Get filtered product list.
+    IMPORTANT: This dynamically loads data to reflect current Streamlit configs/dates.
+    """
+    try:
+        # Always get fresh filtered products to reflect current configs
+        from src.demand_filtering import get_shared_filter_instance
+        filter_instance = get_shared_filter_instance()
+        
+        # Force reload data to pick up new dates/configs
+        filter_instance.load_data(force_reload=True)
+        
+        product_list = filter_instance.get_filtered_product_list()
+        print(f"📦 FRESH FILTERED PRODUCTS: {len(product_list)} products ready for optimization")
+        print(f"🎯 Products: {product_list}")
+        return product_list
+    except Exception as e:
+        print(f"Error loading dynamic product list: {e}")
+        # Fallback to unfiltered list
+        product_list = transformed_data.get_released_product_list(start_date)
+        print(f"📦 FALLBACK UNFILTERED PRODUCTS: {len(product_list)} products -> {product_list}")
+        return product_list
+
+# DO NOT load at import time - always call get_product_list() dynamically
+# PRODUCT_LIST = get_product_list()  # REMOVED - was causing stale data!
 
 
 def get_employee_type_list():
@@ -239,33 +255,29 @@ def get_line_cnt_per_type():
 LINE_CNT_PER_TYPE = get_line_cnt_per_type()
 print("line cnt per type",LINE_CNT_PER_TYPE)
 
-def get_demand_dictionary():
-    try:
-        # Try to get from streamlit session state (from config page)
-        import streamlit as st
-        if hasattr(st, 'session_state') and 'demand_dictionary' in st.session_state:
-            print(f"Using demand dictionary from config page: {len(st.session_state.demand_dictionary)} products")
-            return st.session_state.demand_dictionary
-    except Exception as e:
-        print(f"Could not get demand dictionary from streamlit session: {e}")
-    
-    print(f"Loading demand values from data files")
-    
-    # Try to get filtered demand (only for products ready for optimization)
+def get_demand_dictionary(force_reload=False):
+    """
+    Get filtered demand dictionary. 
+    IMPORTANT: This dynamically loads data to reflect current Streamlit configs/dates.
+    """
     try:
-        from src.demand_filtering import get_filtered_demand_dictionary
-        demand_dictionary = get_filtered_demand_dictionary()
-        print(f"📈 FILTERED DEMAND: {len(demand_dictionary)} products with total demand {sum(demand_dictionary.values())}")
-        return demand_dictionary
-    except ImportError:
-        # Fallback to unfiltered demand if demand_filtering not available
-        demand_df = extract.read_orders_data(start_date=start_date)
-        demand_dictionary = demand_df.groupby('Material Number')["Order quantity (GMEIN)"].sum().to_dict()
-        print(f"📈 UNFILTERED DEMAND: {len(demand_dictionary)} products with total demand {sum(demand_dictionary.values())}")
+        # Always get fresh filtered demand to reflect current configs
+        from src.demand_filtering import get_shared_filter_instance
+        filter_instance = get_shared_filter_instance()
+        
+        # Force reload data to pick up new dates/configs
+        filter_instance.load_data(force_reload=True)
+        
+        demand_dictionary = filter_instance.get_filtered_demand_dictionary()
+        print(f"📈 FRESH FILTERED DEMAND: {len(demand_dictionary)} products with total demand {sum(demand_dictionary.values())}")
+        print(f"🔄 LOADED DYNAMICALLY: Reflects current Streamlit configs")
         return demand_dictionary
+    except Exception as e:
+        print(f"Error loading dynamic demand dictionary: {e}")
+        raise Exception("Demand dictionary not found with error:"+str(e))
 
-DEMAND_DICTIONARY = get_demand_dictionary()
-print(f"🎯 FINAL DEMAND: {DEMAND_DICTIONARY}")
+# DO NOT load at import time - always call get_demand_dictionary() dynamically
+# DEMAND_DICTIONARY = get_demand_dictionary()  # REMOVED - was causing stale data!
 
 def get_cost_list_per_emp_shift():
     try:
@@ -304,11 +316,14 @@ COST_LIST_PER_EMP_SHIFT = get_cost_list_per_emp_shift()
 
 
 
-def get_team_requirements(PRODUCT_LIST):
+def get_team_requirements(product_list=None):
     """
     Extract team requirements from Kits Calculation CSV.
     Returns dictionary with employee type as key and product requirements as nested dict.
     """
+    if product_list is None:
+        product_list = get_product_list()  # Get fresh product list
+        
     try:
         # Check if streamlit has this data (for future extension)
         # streamlit_team_req = dashboard.team_requirements
@@ -330,7 +345,7 @@ def get_team_requirements(PRODUCT_LIST):
     }
     
     # Process each product in the product list
-    for product in PRODUCT_LIST:
+    for product in product_list:
         print("product",product)
         print(f"Processing team requirements for product: {product}")
         product_data = kits_df[kits_df['Kit'] == product]
@@ -349,8 +364,8 @@ def get_team_requirements(PRODUCT_LIST):
     
     return team_req_dict
 
-TEAM_REQ_PER_PRODUCT = get_team_requirements(PRODUCT_LIST)
-print("team requirements per product:", TEAM_REQ_PER_PRODUCT)
+# DO NOT load at import time - always call get_team_requirements() dynamically
+# TEAM_REQ_PER_PRODUCT = get_team_requirements(PRODUCT_LIST)  # REMOVED - was causing stale data!
 
 
 def get_max_employee_per_type_on_day():
@@ -380,7 +395,52 @@ print("max employee per type on day",MAX_EMPLOYEE_PER_TYPE_ON_DAY)
 # available employee but for fixed in shift 1, it is mandatory employment
 
 MAX_HOUR_PER_PERSON_PER_DAY = 14  # legal standard
-MAX_HOUR_PER_SHIFT_PER_PERSON = DefaultConfig.MAX_HOUR_PER_SHIFT_PER_PERSON
+def get_max_hour_per_shift_per_person():
+    """Get max hours per shift per person - checks Streamlit session state first"""
+    try:
+        import streamlit as st
+        if hasattr(st, 'session_state') and 'max_hour_per_shift_per_person' in st.session_state:
+            return st.session_state.max_hour_per_shift_per_person
+    except Exception as e:
+        print(f"Could not get max hours per shift from session: {e}")
+    
+    # Fallback to default only if not configured by user
+    return DefaultConfig.MAX_HOUR_PER_SHIFT_PER_PERSON
+
+MAX_HOUR_PER_SHIFT_PER_PERSON = get_max_hour_per_shift_per_person()
+
+# Removed unnecessary getter functions - use direct imports instead:
+# - MAX_HOUR_PER_PERSON_PER_DAY
+# - MAX_HOUR_PER_SHIFT_PER_PERSON  
+# - KIT_LINE_MATCH_DICT
+# - MAX_PARALLEL_WORKERS
+# - DAILY_WEEKLY_SCHEDULE
+# - EVENING_SHIFT_MODE
+
+# Keep these complex getters that access DefaultConfig or have complex logic:
+def get_evening_shift_demand_threshold():
+    """Get evening shift demand threshold - checks Streamlit session state first"""
+    try:
+        import streamlit as st
+        if hasattr(st, 'session_state') and 'evening_shift_demand_threshold' in st.session_state:
+            return st.session_state.evening_shift_demand_threshold
+    except Exception as e:
+        print(f"Could not get evening shift threshold from session: {e}")
+    
+    # Fallback to default only if not configured by user
+    return getattr(DefaultConfig, 'EVENING_SHIFT_DEMAND_THRESHOLD', 10000)
+
+def get_fixed_min_unicef_per_day():
+    """Get fixed minimum UNICEF staff per day - checks Streamlit session state first"""
+    try:
+        import streamlit as st
+        if hasattr(st, 'session_state') and 'fixed_min_unicef_per_day' in st.session_state:
+            return st.session_state.fixed_min_unicef_per_day
+    except Exception as e:
+        print(f"Could not get fixed min UNICEF from session: {e}")
+    
+    # Fallback to default only if not configured by user
+    return getattr(DefaultConfig, 'FIXED_MIN_UNICEF_PER_DAY', {1: 1, 2: 1, 3: 1, 4: 1, 5: 1})
 def get_per_product_speed():
     try:
         # Try to get from streamlit session state (from config page)
@@ -419,7 +479,19 @@ def get_kit_hierarchy_data():
 KIT_LEVELS, KIT_DEPENDENCIES, PRODUCTION_PRIORITY_ORDER = get_kit_hierarchy_data()
 print(f"Kit Hierarchy loaded: {len(KIT_LEVELS)} kits, Priority order: {len(PRODUCTION_PRIORITY_ORDER)} items")
 
-MAX_PARALLEL_WORKERS = DefaultConfig.MAX_PARALLEL_WORKERS
+def get_max_parallel_workers():
+    """Get max parallel workers - checks Streamlit session state first"""
+    try:
+        import streamlit as st
+        if hasattr(st, 'session_state') and 'max_parallel_workers' in st.session_state:
+            return st.session_state.max_parallel_workers
+    except Exception as e:
+        print(f"Could not get max parallel workers from session: {e}")
+    
+    # Fallback to default only if not configured by user
+    return DefaultConfig.MAX_PARALLEL_WORKERS
+
+MAX_PARALLEL_WORKERS = get_max_parallel_workers()
 # maximum number of workers that can work on a line at the same time
 
 DAILY_WEEKLY_SCHEDULE = "daily"  # daily or weekly ,this needs to be implementedin  in         if F_x1_day is not None... F_x1_week is not None... also need to change x1 to Fixedstaff_first_shift

src/demand_filtering.py

@@ -26,12 +26,19 @@ class DemandFilter:
         self.line_assignments = None
         self.team_requirements = None
         
-    def load_data(self):
+    def load_data(self, force_reload=False):
         """Load all necessary data for filtering"""
         try:
-            # Get start date for demand data
-            from datetime import datetime
-            start_date = datetime(2025, 7, 7)  # Default date, can be made configurable
+            # Skip loading if data already exists and not forcing reload
+            if not force_reload and self.demand_data is not None:
+                print("📊 Using cached filter data (set force_reload=True to refresh)")
+                return True
+                
+            print("🔄 Loading fresh filtering data...")
+            # Get start date for demand data from optimization config
+            from src.config.optimization_config import get_date_span
+            date_span, start_date, end_date = get_date_span()
+            print(f"🗓️ DEMAND FILTERING DATE: Using {start_date.date() if start_date else 'None'} (same as optimization)")
             
             # Load demand data directly from extract
             demand_df = extract.read_orders_data(start_date=start_date)
@@ -204,6 +211,24 @@ class DemandFilter:
             for reason, count in reason_counts.items():
                 print(f"  • {reason}: {count} products")
         
+        # Print data quality warnings for included products (without recursion)
+        # Load speed data for validation
+        speed_data = None
+        try:
+            from src.config import optimization_config
+            speed_data = optimization_config.PER_PRODUCT_SPEED
+        except Exception as e:
+            print(f"Warning: Could not load speed data for validation: {e}")
+        
+        if speed_data:
+            included_without_speed = sum(1 for pid in included_products if pid not in speed_data)
+            if included_without_speed > 0:
+                print(f"\n⚠️  DATA QUALITY WARNING: {included_without_speed} included products missing speed data (will use default 106.7 units/hour)")
+        
+        included_without_hierarchy = sum(1 for pid in included_products if self.classify_product_type(pid)[0] == "unclassified")
+        if included_without_hierarchy > 0:
+            print(f"⚠️  DATA QUALITY WARNING: {included_without_hierarchy} included products missing hierarchy data")
+        
         return included_products, included_demand, excluded_products, excluded_demand
     
     def get_filtered_product_list(self) -> List[str]:
@@ -223,6 +248,14 @@ class DemandFilter:
         all_products = {**included_demand, **excluded_demand}
         product_details = {}
         
+        # Load speed data for additional validation
+        speed_data = None
+        try:
+            from src.config import optimization_config
+            speed_data = optimization_config.PER_PRODUCT_SPEED
+        except Exception as e:
+            print(f"Warning: Could not load speed data for analysis: {e}")
+        
         for product_id, demand in all_products.items():
             product_type, is_standalone_master = self.classify_product_type(product_id)
             is_ready, exclusion_reasons = self.is_product_ready_for_optimization(product_id)
@@ -234,6 +267,9 @@ class DemandFilter:
             # Get line assignment
             line_assignment = self.line_assignments.get(product_id)
             
+            # Get production speed info
+            has_speed_data = speed_data is not None and product_id in speed_data
+            
             product_details[product_id] = {
                 'demand': demand,
                 'product_type': product_type,
@@ -246,9 +282,14 @@ class DemandFilter:
                 'line_assignment': line_assignment,
                 'has_line_assignment': line_assignment is not None,
                 'has_staffing': (unicef_staff + humanizer_staff) > 0,
-                'has_hierarchy': product_type != "unclassified"
+                'has_hierarchy': product_type != "unclassified",
+                'has_speed_data': has_speed_data
             }
         
+        # Calculate data quality statistics for included products
+        included_without_speed = sum(1 for pid in included_products if not product_details[pid]['has_speed_data'])
+        included_without_hierarchy = sum(1 for pid in included_products if not product_details[pid]['has_hierarchy'])
+        
         return {
             'included_count': len(included_products),
             'included_demand': sum(included_demand.values()),
@@ -259,7 +300,10 @@ class DemandFilter:
             'product_details': product_details,
             'standalone_masters_count': sum(1 for p in product_details.values() if p['is_standalone_master']),
             'included_products': included_products,
-            'excluded_products': excluded_products
+            'excluded_products': excluded_products,
+            # Data quality metrics for included products
+            'included_missing_speed_count': included_without_speed,
+            'included_missing_hierarchy_count': included_without_hierarchy
         }
     
     def get_exclusion_summary(self) -> Dict:
@@ -302,6 +346,22 @@ def get_exclusion_summary() -> Dict:
     return filter_instance.get_exclusion_summary()
 
 
+def get_complete_analysis() -> Dict:
+    """Get complete product analysis including data quality metrics"""
+    filter_instance = DemandFilter()
+    return filter_instance.get_complete_product_analysis()
+
+
+# Singleton instance for consistency across modules
+_SHARED_FILTER_INSTANCE = None
+
+def get_shared_filter_instance():
+    """Returns a shared singleton instance of DemandFilter."""
+    global _SHARED_FILTER_INSTANCE
+    if _SHARED_FILTER_INSTANCE is None:
+        _SHARED_FILTER_INSTANCE = DemandFilter()
+    return _SHARED_FILTER_INSTANCE
+
 if __name__ == "__main__":
     # Test the filtering
     filter_instance = DemandFilter()

src/demand_validation.py

@@ -1,435 +0,0 @@
-#!/usr/bin/env python3
-"""
-Demand Data Validation Module
-
-This module provides comprehensive validation and analysis of demand data,
-including product classification, hierarchy analysis, line assignments,
-and staffing requirements.
-"""
-
-import pandas as pd
-import streamlit as st
-from typing import Dict, List, Tuple, Optional
-import json
-from src.config.constants import ShiftType, LineType, KitLevel
-
-
-class DemandValidator:
-    """
-    Validates and analyzes demand data for optimization readiness
-    """
-    
-    def __init__(self):
-        self.demand_data = None
-        self.hierarchy_data = None
-        self.team_requirements = None
-        self.line_assignments = None
-        self.speed_data = None
-        self.validation_results = {}
-        
-    def load_data(self):
-        """Load all necessary data for validation"""
-        try:
-            # Load demand data
-            from src.config import optimization_config
-            self.demand_data = optimization_config.get_demand_dictionary()
-            
-            # Load hierarchy data
-            try:
-                with open('data/hierarchy_exports/kit_hierarchy.json', 'r') as f:
-                    self.hierarchy_data = json.load(f)
-            except FileNotFoundError:
-                print("Warning: Kit hierarchy file not found. Some analysis will be limited.")
-                self.hierarchy_data = {}
-            
-            # Load team requirements
-            self.team_requirements = optimization_config.TEAM_REQ_PER_PRODUCT
-            
-            # Load line assignments
-            self.line_assignments = optimization_config.KIT_LINE_MATCH_DICT
-            
-            # Load speed data
-            self.speed_data = optimization_config.PER_PRODUCT_SPEED
-            
-            # Load kit levels and dependencies
-            self.kit_levels = optimization_config.KIT_LEVELS
-            self.kit_dependencies = optimization_config.KIT_DEPENDENCIES
-            
-            return True
-            
-        except Exception as e:
-            error_msg = f"Error loading data: {str(e)}"
-            print(error_msg)
-            # Only use st.error if we're in a Streamlit context
-            try:
-                st.error(error_msg)
-            except:
-                pass
-            return False
-    
-    def classify_product_type(self, product_id: str) -> Tuple[str, str]:
-        """
-        Classify product as master/subkit/prepack
-        Returns: (type, level_name)
-        """
-        if product_id in self.kit_levels:
-            level = self.kit_levels[product_id]
-            level_name = KitLevel.get_name(level)
-            
-            if level == 0:
-                return "prepack", level_name
-            elif level == 1:
-                return "subkit", level_name
-            elif level == 2:
-                return "master", level_name
-            else:
-                return "unknown", f"level_{level}"
-        else:
-            return "unclassified", "no_hierarchy_data"
-    
-    def is_standalone_master(self, product_id: str) -> bool:
-        """
-        Check if this is a standalone master (master with no subkits/prepacks)
-        """
-        product_type, _ = self.classify_product_type(product_id)
-        
-        if product_type != "master":
-            return False
-        
-        # Check if this master has any dependencies
-        dependencies = self.kit_dependencies.get(product_id, [])
-        return len(dependencies) == 0
-    
-    def get_line_assignment(self, product_id: str) -> Tuple[Optional[int], str]:
-        """
-        Get line assignment for product
-        Returns: (line_type_id, line_name)
-        """
-        if product_id in self.line_assignments:
-            line_type = self.line_assignments[product_id]
-            line_name = LineType.get_name(line_type)
-            return line_type, line_name
-        else:
-            return None, "no_assignment"
-    
-    def get_staffing_requirements(self, product_id: str) -> Dict[str, int]:
-        """
-        Get staffing requirements for product
-        Returns: {employee_type: count}
-        """
-        staffing = {}
-        
-        for emp_type, products in self.team_requirements.items():
-            staffing[emp_type] = products.get(product_id, 0)
-        
-        return staffing
-    
-    def get_production_speed(self, product_id: str) -> Optional[float]:
-        """Get production speed for product"""
-        return self.speed_data.get(product_id, None)
-    
-    def get_dependencies(self, product_id: str) -> List[str]:
-        """Get list of dependencies for product"""
-        return self.kit_dependencies.get(product_id, [])
-    
-    def validate_all_products(self) -> pd.DataFrame:
-        """
-        Perform comprehensive validation of all products with demand
-        Returns: DataFrame with validation results
-        """
-        results = []
-        
-        for product_id, demand in self.demand_data.items():
-            if demand > 0:  # Only analyze products with actual demand
-                
-                # Basic classification
-                product_type, level_name = self.classify_product_type(product_id)
-                is_standalone = self.is_standalone_master(product_id)
-                
-                # Line assignment
-                line_type_id, line_name = self.get_line_assignment(product_id)
-                
-                # Staffing requirements
-                staffing = self.get_staffing_requirements(product_id)
-                unicef_staff = staffing.get('UNICEF Fixed term', 0)
-                humanizer_staff = staffing.get('Humanizer', 0)
-                total_staff = unicef_staff + humanizer_staff
-                
-                # Production speed
-                speed = self.get_production_speed(product_id)
-                
-                # Dependencies
-                dependencies = self.get_dependencies(product_id)
-                dependency_count = len(dependencies)
-                
-                # Calculate production time needed (if speed available)
-                production_hours_needed = None
-                if speed and speed > 0:
-                    production_hours_needed = demand / speed
-                
-                # Validation flags
-                has_line_assignment = line_type_id is not None
-                has_staffing_data = total_staff > 0
-                has_speed_data = speed is not None
-                has_hierarchy_data = product_type != "unclassified"
-                
-                # Overall validation status
-                validation_issues = []
-                if not has_line_assignment:
-                    validation_issues.append("no_line_assignment")
-                if not has_staffing_data:
-                    validation_issues.append("no_staffing_data")
-                if not has_speed_data:
-                    validation_issues.append("no_speed_data")
-                if not has_hierarchy_data:
-                    validation_issues.append("no_hierarchy_data")
-                
-                validation_status = "✅ Valid" if not validation_issues else f"⚠️ Issues: {', '.join(validation_issues)}"
-                
-                results.append({
-                    'Product ID': product_id,
-                    'Demand': demand,
-                    'Product Type': product_type.title(),
-                    'Level': level_name,
-                    'Is Standalone Master': "Yes" if is_standalone else "No",
-                    'Line Type ID': line_type_id if line_type_id else "N/A",
-                    'Line Type': line_name,
-                    'UNICEF Staff': unicef_staff,
-                    'Humanizer Staff': humanizer_staff,
-                    'Total Staff': total_staff,
-                    'Production Speed (units/hour)': f"{speed:.1f}" if speed else "N/A",
-                    'Production Hours Needed': f"{production_hours_needed:.1f}" if production_hours_needed else "N/A",
-                    'Dependencies Count': dependency_count,
-                    'Dependencies': ", ".join(dependencies[:3]) + ("..." if len(dependencies) > 3 else ""),
-                    'Has Line Assignment': "✅" if has_line_assignment else "❌",
-                    'Has Staffing Data': "✅" if has_staffing_data else "❌", 
-                    'Has Speed Data': "✅" if has_speed_data else "❌",
-                    'Has Hierarchy Data': "✅" if has_hierarchy_data else "❌",
-                    'Validation Status': validation_status
-                })
-        
-        df = pd.DataFrame(results)
-        
-        # Sort by validation issues first, then by demand
-        df['Has Issues'] = df['Validation Status'].str.contains('Issues')
-        df = df.sort_values(['Has Issues', 'Demand'], ascending=[False, False])
-        df = df.drop('Has Issues', axis=1)
-        
-        return df
-    
-    def get_summary_statistics(self, df: pd.DataFrame) -> Dict:
-        """Generate summary statistics from validation results"""
-        
-        total_products = len(df)
-        total_demand = df['Demand'].sum()
-        
-        # Product type distribution
-        type_counts = df['Product Type'].value_counts().to_dict()
-        
-        # Validation status
-        valid_products = len(df[df['Validation Status'] == "✅ Valid"])
-        invalid_products = total_products - valid_products
-        
-        # Line assignment issues
-        no_line_assignment = len(df[df['Has Line Assignment'] == "❌"])
-        
-        # Staffing issues  
-        no_staffing = len(df[df['Has Staffing Data'] == "❌"])
-        
-        # Speed data issues
-        no_speed = len(df[df['Has Speed Data'] == "❌"])
-        
-        # Hierarchy issues
-        no_hierarchy = len(df[df['Has Hierarchy Data'] == "❌"])
-        
-        # Standalone masters
-        standalone_masters = len(df[(df['Product Type'] == 'Master') & (df['Is Standalone Master'] == 'Yes')])
-        
-        # Staffing summary
-        total_unicef_needed = df['UNICEF Staff'].sum()
-        total_humanizer_needed = df['Humanizer Staff'].sum()
-        
-        return {
-            'total_products': total_products,
-            'total_demand': total_demand,
-            'valid_products': valid_products,
-            'invalid_products': invalid_products,
-            'type_counts': type_counts,
-            'no_line_assignment': no_line_assignment,
-            'no_staffing': no_staffing,
-            'no_speed': no_speed,
-            'no_hierarchy': no_hierarchy,
-            'standalone_masters': standalone_masters,
-            'total_unicef_needed': total_unicef_needed,
-            'total_humanizer_needed': total_humanizer_needed
-        }
-
-
-def display_demand_validation():
-    """
-    Display demand validation analysis in Streamlit
-    """
-    st.header("📋 Demand Data Validation")
-    st.markdown("Comprehensive analysis of products with demand to identify potential optimization issues.")
-    
-    # Initialize validator
-    validator = DemandValidator()
-    
-    # Load data
-    with st.spinner("Loading data for validation..."):
-        if not validator.load_data():
-            st.error("Failed to load data for validation.")
-            return
-    
-    # Perform validation
-    with st.spinner("Analyzing demand data..."):
-        validation_df = validator.validate_all_products()
-        summary_stats = validator.get_summary_statistics(validation_df)
-    
-    # Display summary statistics
-    st.subheader("📊 Summary Statistics")
-    
-    col1, col2, col3, col4 = st.columns(4)
-    
-    with col1:
-        st.metric("Total Products", summary_stats['total_products'])
-        st.metric("Valid Products", summary_stats['valid_products'])
-    
-    with col2:
-        st.metric("Total Demand", f"{summary_stats['total_demand']:,}")
-        st.metric("Invalid Products", summary_stats['invalid_products'])
-    
-    with col3:
-        st.metric("UNICEF Staff Needed", summary_stats['total_unicef_needed'])
-        st.metric("Standalone Masters", summary_stats['standalone_masters'])
-    
-    with col4:
-        st.metric("Humanizer Staff Needed", summary_stats['total_humanizer_needed'])
-        st.metric("No Line Assignment", summary_stats['no_line_assignment'])
-    
-    # Product type distribution
-    st.subheader("📈 Product Type Distribution")
-    if summary_stats['type_counts']:
-        col1, col2 = st.columns(2)
-        
-        with col1:
-            type_df = pd.DataFrame(list(summary_stats['type_counts'].items()), 
-                                 columns=['Product Type', 'Count'])
-            st.bar_chart(type_df.set_index('Product Type'))
-        
-        with col2:
-            for ptype, count in summary_stats['type_counts'].items():
-                percentage = (count / summary_stats['total_products']) * 100
-                st.write(f"**{ptype}:** {count} products ({percentage:.1f}%)")
-    
-    # Validation issues summary
-    st.subheader("⚠️ Validation Issues Summary")
-    
-    col1, col2, col3, col4 = st.columns(4)
-    
-    with col1:
-        st.metric("No Line Assignment", summary_stats['no_line_assignment'], 
-                 delta=None if summary_stats['no_line_assignment'] == 0 else "Issue")
-    
-    with col2:
-        st.metric("No Staffing Data", summary_stats['no_staffing'],
-                 delta=None if summary_stats['no_staffing'] == 0 else "Issue")
-    
-    with col3:
-        st.metric("No Speed Data", summary_stats['no_speed'],
-                 delta=None if summary_stats['no_speed'] == 0 else "Issue")
-    
-    with col4:
-        st.metric("No Hierarchy Data", summary_stats['no_hierarchy'],
-                 delta=None if summary_stats['no_hierarchy'] == 0 else "Issue")
-    
-    # Detailed validation results
-    st.subheader("📋 Detailed Validation Results")
-    
-    # Filter options
-    col1, col2, col3 = st.columns(3)
-    
-    with col1:
-        show_only_issues = st.checkbox("Show only products with issues", value=False)
-    
-    with col2:
-        product_type_filter = st.selectbox("Filter by product type", 
-                                         options=["All"] + list(summary_stats['type_counts'].keys()))
-    
-    with col3:
-        min_demand = st.number_input("Minimum demand", min_value=0, value=0)
-    
-    # Apply filters
-    filtered_df = validation_df.copy()
-    
-    if show_only_issues:
-        filtered_df = filtered_df[filtered_df['Validation Status'] != "✅ Valid"]
-    
-    if product_type_filter != "All":
-        filtered_df = filtered_df[filtered_df['Product Type'] == product_type_filter]
-    
-    if min_demand > 0:
-        filtered_df = filtered_df[filtered_df['Demand'] >= min_demand]
-    
-    # Display filtered results
-    st.write(f"Showing {len(filtered_df)} of {len(validation_df)} products")
-    
-    if len(filtered_df) > 0:
-        # Configure column display
-        column_config = {
-            'Product ID': st.column_config.TextColumn('Product ID', width="medium"),
-            'Demand': st.column_config.NumberColumn('Demand', format="%d"),
-            'Production Speed (units/hour)': st.column_config.TextColumn('Speed', width="small"),
-            'Production Hours Needed': st.column_config.TextColumn('Hours Needed', width="small"),
-            'Validation Status': st.column_config.TextColumn('Status', width="large"),
-        }
-        
-        st.dataframe(
-            filtered_df,
-            column_config=column_config,
-            use_container_width=True,
-            height=400
-        )
-        
-        # Export option
-        if st.button("📥 Export Validation Results to CSV"):
-            csv = validation_df.to_csv(index=False)
-            st.download_button(
-                label="Download CSV",
-                data=csv,
-                file_name="demand_validation_results.csv",
-                mime="text/csv"
-            )
-    else:
-        st.info("No products match the selected filters.")
-    
-    # Recommendations
-    st.subheader("💡 Recommendations")
-    
-    recommendations = []
-    
-    if summary_stats['no_line_assignment'] > 0:
-        recommendations.append(f"**Line Assignment**: {summary_stats['no_line_assignment']} products need line type assignments in the kit-line mapping.")
-    
-    if summary_stats['no_staffing'] > 0:
-        recommendations.append(f"**Staffing Data**: {summary_stats['no_staffing']} products are missing staffing requirements.")
-    
-    if summary_stats['no_speed'] > 0:
-        recommendations.append(f"**Speed Data**: {summary_stats['no_speed']} products are missing production speed data.")
-    
-    if summary_stats['no_hierarchy'] > 0:
-        recommendations.append(f"**Hierarchy Data**: {summary_stats['no_hierarchy']} products are not in the kit hierarchy.")
-    
-    if summary_stats['standalone_masters'] > 0:
-        recommendations.append(f"**Standalone Masters**: {summary_stats['standalone_masters']} master kits have no dependencies (verify if this is intended).")
-    
-    if recommendations:
-        for rec in recommendations:
-            st.warning(rec)
-    else:
-        st.success("✅ All validation checks passed! Your demand data appears to be complete and ready for optimization.")
-
-
-if __name__ == "__main__":
-    # For testing
-    display_demand_validation()

src/demand_validation_viz.py

@@ -17,7 +17,8 @@ from src.demand_filtering import DemandFilter
 
 class DemandValidationViz:
     """
-    Provides visualization for demand data validation using the filtering module
+    Provides visualization for demand data validation using the filtering module.
+    This class focuses purely on displaying the results from demand filtering.
     """
     
     def __init__(self):
@@ -43,40 +44,7 @@ class DemandValidationViz:
                 pass
             return False
     
-    def classify_product_type(self, product_id: str) -> Tuple[str, str]:
-        """Get product classification from filter instance"""
-        if product_id in self.filter_instance.kit_levels:
-            level = self.filter_instance.kit_levels[product_id]
-            level_name = KitLevel.get_name(level)
-            
-            if level == 0:
-                return "prepack", level_name
-            elif level == 1:
-                return "subkit", level_name
-            elif level == 2:
-                return "master", level_name
-            else:
-                return "unknown", f"level_{level}"
-        else:
-            return "unclassified", "no_hierarchy_data"
-    
-    def get_line_assignment(self, product_id: str) -> Tuple[Optional[int], str]:
-        """Get line assignment from filter instance"""
-        if product_id in self.filter_instance.line_assignments:
-            line_type = self.filter_instance.line_assignments[product_id]
-            line_name = LineType.get_name(line_type)
-            return line_type, line_name
-        else:
-            return None, "no_assignment"
-    
-    def get_staffing_requirements(self, product_id: str) -> Dict[str, int]:
-        """Get staffing requirements from filter instance"""
-        staffing = {}
-        
-        for emp_type, products in self.filter_instance.team_requirements.items():
-            staffing[emp_type] = products.get(product_id, 0)
-        
-        return staffing
+    # Remove duplicate methods - use filter_instance data directly
     
     def get_production_speed(self, product_id: str) -> Optional[float]:
         """Get production speed for product"""
@@ -122,21 +90,21 @@ class DemandValidationViz:
                 else:
                     level_name = f"level_{details['product_type']}"
             
-            # Overall status
+            # Overall status with enhanced detail
             if not details['is_included_in_optimization']:
                 validation_status = f"🚫 Excluded: {', '.join(details['exclusion_reasons'])}"
             else:
-                # Check for other issues that don't exclude from optimization
-                other_issues = []
+                # Check for data quality issues that don't exclude from optimization
+                data_quality_issues = []
                 if speed is None:
-                    other_issues.append("no_speed_data")
+                    data_quality_issues.append("missing_speed_data (will use default)")
                 if not details['has_hierarchy']:
-                    other_issues.append("no_hierarchy_data")
+                    data_quality_issues.append("no_hierarchy_data")
                 
-                if other_issues:
-                    validation_status = f"⚠️ Issues: {', '.join(other_issues)}"
+                if data_quality_issues:
+                    validation_status = f"⚠️ Data Issues: {', '.join(data_quality_issues)}"
                 else:
-                    validation_status = "✅ Included in optimization"
+                    validation_status = "✅ Ready for optimization"
             
             results.append({
                 'Product ID': product_id,
@@ -153,10 +121,11 @@ class DemandValidationViz:
                 'Production Hours Needed': f"{production_hours_needed:.1f}" if production_hours_needed else "N/A",
                 'Has Line Assignment': "✅" if details['has_line_assignment'] else "❌",
                 'Has Staffing Data': "✅" if details['has_staffing'] else "❌", 
-                'Has Speed Data': "✅" if speed is not None else "❌",
+                'Has Speed Data': "✅" if speed is not None else "❌ (will use default)",
                 'Has Hierarchy Data': "✅" if details['has_hierarchy'] else "❌",
                 'Excluded from Optimization': not details['is_included_in_optimization'],
                 'Exclusion Reasons': ', '.join(details['exclusion_reasons']) if details['exclusion_reasons'] else '',
+                'Data Quality Issues': ', '.join(data_quality_issues) if details['is_included_in_optimization'] and data_quality_issues else '',
                 'Validation Status': validation_status
             })
         
@@ -208,10 +177,12 @@ class DemandValidationViz:
 
 def display_demand_validation():
     """
-    Display demand validation analysis in Streamlit
+    Display demand validation analysis in Streamlit.
+    Shows the results of demand filtering with detailed data quality analysis.
     """
     st.header("📋 Demand Data Validation")
-    st.markdown("Comprehensive analysis of products with demand to identify potential optimization issues.")
+    st.markdown("""**Analysis of Filtered Demand Data**: This page shows which products are included/excluded from optimization 
+    based on the demand filtering criteria, plus data quality assessment for included products.""")
     
     # Initialize validator
     validator = DemandValidationViz()
@@ -265,7 +236,7 @@ def display_demand_validation():
     
     # Validation issues summary for included products
     st.subheader("⚠️ Data Quality Issues (Products Included in Optimization)")
-    st.write("Issues affecting products that **will be** included in optimization:")
+    st.write("Data quality issues affecting products that **will be** included in optimization (these don't exclude products but may affect optimization quality):")
     
     col1, col2, col3, col4 = st.columns(4)
     
@@ -277,9 +248,9 @@ def display_demand_validation():
         st.metric("No Staffing Data", summary_stats['no_staffing'],
                  delta=None if summary_stats['no_staffing'] == 0 else "Issue")
     
-    with col3:
-        st.metric("No Speed Data", summary_stats['no_speed'],
-                 delta=None if summary_stats['no_speed'] == 0 else "Issue")
+        with col3:
+            st.metric("No Speed Data", summary_stats['no_speed'],
+                     delta=None if summary_stats['no_speed'] == 0 else "Will use default")
     
     with col4:
         st.metric("No Hierarchy Data", summary_stats['no_hierarchy'],
@@ -313,7 +284,7 @@ def display_demand_validation():
             filtered_included = filtered_included[filtered_included['Demand'] >= included_min_demand]
         
         # Configure column display for included
-        included_columns = ['Product ID', 'Demand', 'Product Type', 'Line Type', 'UNICEF Staff', 'Humanizer Staff', 'Production Speed (units/hour)', 'Validation Status']
+        included_columns = ['Product ID', 'Demand', 'Product Type', 'Line Type', 'UNICEF Staff', 'Humanizer Staff', 'Production Speed (units/hour)', 'Data Quality Issues', 'Validation Status']
         
         st.dataframe(
             filtered_included[included_columns],
@@ -326,7 +297,10 @@ def display_demand_validation():
     # Products Excluded from Optimization  
     st.subheader("🚫 Products Excluded from Optimization")
     st.write(f"**{len(excluded_df)} products** are excluded from optimization with total demand of **{excluded_df['Demand'].sum():,} units**")
-    st.info("These products are omitted from optimization due to missing line assignments or zero staffing requirements.")
+    st.info("""These products are **filtered out** from optimization due to:
+    • Missing line assignments (for non-standalone masters)
+    • Zero staffing requirements (both UNICEF and Humanizer staff = 0)
+    • Non-standalone masters (excluded from production planning)""")
     
     if len(excluded_df) > 0:
         # Show exclusion breakdown
@@ -373,7 +347,7 @@ def display_demand_validation():
         recommendations.append(f"**Staffing Data**: {summary_stats['no_staffing']} products included in optimization are missing staffing requirements.")
     
     if summary_stats['no_speed'] > 0:
-        recommendations.append(f"**Speed Data**: {summary_stats['no_speed']} products included in optimization are missing production speed data (will use defaults).")
+        recommendations.append(f"**Speed Data**: {summary_stats['no_speed']} products included in optimization are missing production speed data. The optimization will use a default speed of 106.7 units/hour for these products.")
     
     if summary_stats['no_hierarchy'] > 0:
         recommendations.append(f"**Hierarchy Data**: {summary_stats['no_hierarchy']} products included in optimization are not in the kit hierarchy.")
@@ -384,9 +358,11 @@ def display_demand_validation():
     
     # Overall status
     if summary_stats['included_products'] > 0:
-        st.success(f"✅ **Ready for Optimization**: {summary_stats['included_products']} products with {summary_stats['included_demand']:,} units demand are ready for optimization!")
+        st.success(f"✅ **Optimization Scope**: {summary_stats['included_products']} products with {summary_stats['included_demand']:,} units demand will be included in optimization!")
+        if summary_stats['no_speed'] == 0 and summary_stats['no_hierarchy'] == 0:
+            st.info("🎉 All included products have complete data - optimization should run smoothly!")
     else:
-        st.error("❌ No products are ready for optimization. Please review the exclusion criteria above.")
+        st.error("❌ No products passed the filtering criteria. Please review the exclusion reasons above and check your data configuration.")
 
 
 if __name__ == "__main__":

src/models/optimizer_real.py

@@ -14,12 +14,12 @@ from src.config.constants import ShiftType, LineType, KitLevel
 # ---- config import (프로젝트 경로에 맞춰 조정) ----
 from src.config.optimization_config import (
     DATE_SPAN,                # [1..N]
-    PRODUCT_LIST,             # list of products (e.g., ['A','B',...])
+    get_product_list,         # DYNAMIC: list of products (e.g., ['A','B',...])
     EMPLOYEE_TYPE_LIST,       # e.g., ['UNICEF Fixed term','Humanizer']
     SHIFT_LIST,               # e.g., [1,2,3]
     LINE_LIST,                # e.g., [6,7]  (line type ids)
     LINE_CNT_PER_TYPE,        # {6: count_of_long_lines, 7: count_of_short_lines}
-    DEMAND_DICTIONARY,        # {product: total_units_over_period}
+    get_demand_dictionary,    # DYNAMIC: {product: total_units_over_period}
     COST_LIST_PER_EMP_SHIFT,  # {emp_type: {shift: cost_per_hour}}
     MAX_EMPLOYEE_PER_TYPE_ON_DAY,    # {emp_type: {t: headcount}}
     MAX_HOUR_PER_PERSON_PER_DAY,     # e.g., 14
@@ -28,7 +28,7 @@ from src.config.optimization_config import (
     MAX_PARALLEL_WORKERS,            # {6: max_workers, 7: max_workers}
     DAILY_WEEKLY_SCHEDULE,           # 'daily' or 'weekly' 
     FIXED_STAFF_CONSTRAINT_MODE,     # not used in fixed-team model (동시 투입이라 무의미)
-    TEAM_REQ_PER_PRODUCT,            # {emp_type: {product: team_size}} from Kits_Calculation.csv
+    get_team_requirements,    # DYNAMIC: {emp_type: {product: team_size}} from Kits_Calculation.csv
     PAYMENT_MODE_CONFIG,             # {shift: 'bulk'/'partial'} payment mode configuration
     KIT_LINE_MATCH_DICT,
     EVENING_SHIFT_MODE,
@@ -51,7 +51,7 @@ KIT_LINE_MATCH_DICT
 print("KIT_LINE_MATCH_DICT",KIT_LINE_MATCH_DICT)
 
 # 3) If specific product is not produced on specific date, set it to 0
-ACTIVE = {t: {p: 1 for p in PRODUCT_LIST} for t in DATE_SPAN}
+# ACTIVE will be built dynamically in solve function based on fresh PRODUCT_LIST
 # Example: ACTIVE[2]['C'] = 0  # Disable product C on day 2
 
 
@@ -159,11 +159,29 @@ def sort_products_by_hierarchy(product_list):
 # Dependency ordering is now handled by topological sorting in sort_products_by_hierarchy()
 
 def solve_fixed_team_weekly():
+    # *** CRITICAL: Load fresh data to reflect current Streamlit configs ***
+    print("\n" + "="*60)
+    print("🔄 LOADING FRESH DATA FOR OPTIMIZATION")
+    print("="*60)
+    
+    # Get fresh product list and demand data
+    PRODUCT_LIST = get_product_list()
+    DEMAND_DICTIONARY = get_demand_dictionary()
+    TEAM_REQ_PER_PRODUCT = get_team_requirements(PRODUCT_LIST)
+    
+    print(f"📦 LOADED PRODUCTS: {len(PRODUCT_LIST)} products")
+    print(f"📈 LOADED DEMAND: {sum(DEMAND_DICTIONARY.values())} total units")
+    print(f"👥 LOADED TEAM REQUIREMENTS: {len(TEAM_REQ_PER_PRODUCT)} employee types")
+    
+    # Build ACTIVE schedule for fresh product list
+    ACTIVE = {t: {p: 1 for p in PRODUCT_LIST} for t in DATE_SPAN}
+    
     # --- Sets ---
     D = list(DATE_SPAN)
+    # print("D",D)
     S = sorted(list(SHIFT_LIST))
     E = list(EMPLOYEE_TYPE_LIST)  # e.g., ['UNICEF Fixed term','Humanizer']
-    
+    print("E",E)
     # *** HIERARCHY SORTING: Sort products by production priority ***
     print("\n" + "="*60)
     print("🔗 APPLYING HIERARCHY-BASED PRODUCTION ORDERING")
@@ -180,7 +198,7 @@ def solve_fixed_team_weekly():
     N_day = MAX_EMPLOYEE_PER_TYPE_ON_DAY              # {emp_type:{t:headcount}}
     cost = COST_LIST_PER_EMP_SHIFT                    # {emp_type:{shift:cost}}
     d_week = DEMAND_DICTIONARY                        # {product: demand over period}
-
+    print("d_week",d_week)
     # --- Feasibility quick checks ---
     
     # 1) If team size is greater than max_workers_line, block the product-line type combination