data validation in progress and hierarchy parser fixed

app.py

@@ -61,7 +61,7 @@ elif page == "📊 Optimization Results":
 elif page == "📋 Demand Validation":
     # Import and render the demand validation page
     try:
-        from src.demand_validation import display_demand_validation
+        from src.demand_validation_viz import display_demand_validation
         
         st.title("📋 Demand Data Validation")
         st.markdown("---")

config_page.py

@@ -64,7 +64,7 @@ def render_config_page():
         st.markdown("Analyzing your demand data to identify potential optimization issues...")
         
         try:
-            from src.demand_validation import display_demand_validation
+            from src.demand_validation_viz import display_demand_validation
             display_demand_validation()
             
             # Show validation reminder before optimization
@@ -995,10 +995,10 @@ def check_critical_data_issues():
         if src_path not in sys.path:
             sys.path.append(src_path)
             
-        from src.demand_validation import DemandValidator
+        from src.demand_validation_viz import DemandValidationViz
         
         # Initialize validator and load data
-        validator = DemandValidator()
+        validator = DemandValidationViz()
         if not validator.load_data():
             warnings.append("Failed to load validation data")
             return warnings

optimization_results.py

@@ -947,7 +947,7 @@ def display_demand_validation_tab():
     Display demand validation in the optimization results tab
     """
     try:
-        from src.demand_validation import display_demand_validation
+        from src.demand_validation_viz import display_demand_validation
         display_demand_validation()
     except ImportError as e:
         st.error(f"❌ Error loading demand validation module: {str(e)}")

src/config/optimization_config.py

@@ -69,8 +69,17 @@ extract.set_global_dates(start_date, end_date)
 
 print(f"\n📅 DATE RANGE: {start_date} to {end_date}")
 print(f"📁 PRODUCT SOURCE: COOIS_Released_Prod_Orders.csv")
-PRODUCT_LIST = transformed_data.get_released_product_list(start_date)
-print(f"📦 PRODUCTS FOUND: {len(PRODUCT_LIST)} products -> {PRODUCT_LIST}")
+
+# Get filtered products ready for optimization
+try:
+    from src.demand_filtering import get_filtered_product_list
+    PRODUCT_LIST = get_filtered_product_list()
+    print(f"📦 FILTERED PRODUCTS: {len(PRODUCT_LIST)} products ready for optimization")
+    print(f"🎯 Products: {PRODUCT_LIST}")
+except ImportError:
+    # Fallback to unfiltered list if demand_filtering not available
+    PRODUCT_LIST = transformed_data.get_released_product_list(start_date)
+    print(f"📦 UNFILTERED PRODUCTS: {len(PRODUCT_LIST)} products -> {PRODUCT_LIST}")
 
 
 def get_employee_type_list():
@@ -182,6 +191,7 @@ def get_kit_line_match():
     line_name_to_id = {
         "long line": LineType.LONG_LINE,
         "mini load": LineType.MINI_LOAD,
+        "miniload": LineType.MINI_LOAD,     # Alternative naming (no space)
         "Long_line": LineType.LONG_LINE,    # Alternative naming
         "Mini_load": LineType.MINI_LOAD,    # Alternative naming
     }
@@ -202,8 +212,8 @@ def get_kit_line_match():
             # Already numeric
             converted_dict[kit] = int(line_name)
         else:
-            # Missing or empty line type - default to long line
-            converted_dict[kit] = LineType.LONG_LINE
+            # Missing or empty line type - skip (no production needed for non-standalone masters)
+            pass  # Don't add to converted_dict - these kits won't have line assignments
             
     return converted_dict
 
@@ -239,12 +249,20 @@ def get_demand_dictionary():
     except Exception as e:
         print(f"Could not get demand dictionary from streamlit session: {e}")
     
-    print(f"Loading default demand values from data files")
-    # Use released orders instead of planned orders for demand
-    demand_df = extract.read_orders_data(start_date=start_date)
-    demand_dictionary = demand_df.groupby('Material Number')["Order quantity (GMEIN)"].sum().to_dict()
-    print(f"📈 DEMAND DATA: {len(demand_dictionary)} products with total demand {sum(demand_dictionary.values())}")
-    return demand_dictionary
+    print(f"Loading demand values from data files")
+    
+    # Try to get filtered demand (only for products ready for optimization)
+    try:
+        from src.demand_filtering import get_filtered_demand_dictionary
+        demand_dictionary = get_filtered_demand_dictionary()
+        print(f"📈 FILTERED DEMAND: {len(demand_dictionary)} products with total demand {sum(demand_dictionary.values())}")
+        return demand_dictionary
+    except ImportError:
+        # Fallback to unfiltered demand if demand_filtering not available
+        demand_df = extract.read_orders_data(start_date=start_date)
+        demand_dictionary = demand_df.groupby('Material Number')["Order quantity (GMEIN)"].sum().to_dict()
+        print(f"📈 UNFILTERED DEMAND: {len(demand_dictionary)} products with total demand {sum(demand_dictionary.values())}")
+        return demand_dictionary
 
 DEMAND_DICTIONARY = get_demand_dictionary()
 print(f"🎯 FINAL DEMAND: {DEMAND_DICTIONARY}")

src/demand_filtering.py

@@ -0,0 +1,312 @@
+"""
+Demand Data Filtering Module
+
+This module filters demand data to include only products that are ready for optimization.
+Excludes products that:
+1. Have no line type assignments (non-standalone masters)
+2. Have zero staffing requirements (both Humanizer and UNICEF staff = 0)
+
+The filtered data is used by the optimization system.
+"""
+
+import pandas as pd
+from typing import Dict, List, Tuple
+from src.etl import extract
+
+
+class DemandFilter:
+    """
+    Filters demand data to include only products ready for optimization
+    """
+    
+    def __init__(self):
+        self.demand_data = None
+        self.kit_levels = None
+        self.kit_dependencies = None
+        self.line_assignments = None
+        self.team_requirements = None
+        
+    def load_data(self):
+        """Load all necessary data for filtering"""
+        try:
+            # Get start date for demand data
+            from datetime import datetime
+            start_date = datetime(2025, 7, 7)  # Default date, can be made configurable
+            
+            # Load demand data directly from extract
+            demand_df = extract.read_orders_data(start_date=start_date)
+            self.demand_data = demand_df.groupby('Material Number')["Order quantity (GMEIN)"].sum().to_dict()
+            
+            # Load kit hierarchy data
+            kit_levels, dependencies, _ = extract.get_production_order_data()
+            self.kit_levels = kit_levels
+            self.kit_dependencies = dependencies
+            
+            # Load line assignments from kit line match data
+            kit_line_match = extract.read_kit_line_match_data()
+            kit_line_match_dict = kit_line_match.set_index("kit_name")["line_type"].to_dict()
+            
+            # Convert string line names to numeric IDs
+            from src.config.constants import LineType
+            line_name_to_id = {
+                "long line": LineType.LONG_LINE,
+                "mini load": LineType.MINI_LOAD,
+                "miniload": LineType.MINI_LOAD,
+                "Long_line": LineType.LONG_LINE,
+                "Mini_load": LineType.MINI_LOAD,
+            }
+            
+            self.line_assignments = {}
+            for kit, line_name in kit_line_match_dict.items():
+                if isinstance(line_name, str) and line_name.strip():
+                    line_id = line_name_to_id.get(line_name.strip())
+                    if line_id is not None:
+                        self.line_assignments[kit] = line_id
+                elif isinstance(line_name, (int, float)) and not pd.isna(line_name):
+                    self.line_assignments[kit] = int(line_name)
+            
+            # Load team requirements from Kits Calculation data
+            kits_df = extract.read_personnel_requirement_data()
+            self.team_requirements = {
+                'UNICEF Fixed term': kits_df.set_index('Kit')['UNICEF staff'].to_dict(),
+                'Humanizer': kits_df.set_index('Kit')['Humanizer'].to_dict()
+            }
+            
+            return True
+            
+        except Exception as e:
+            print(f"Error loading data for filtering: {str(e)}")
+            return False
+    
+    def classify_product_type(self, product_id: str) -> Tuple[str, bool]:
+        """
+        Classify product type and check if it's a standalone master.
+        
+        Returns:
+            Tuple[str, bool]: (product_type, is_standalone_master)
+        """
+        if product_id in self.kit_levels:
+            level = self.kit_levels[product_id]
+            
+            if level == 0:
+                return "prepack", False
+            elif level == 1:
+                return "subkit", False
+            elif level == 2:
+                # Check if this master is standalone (no subkits/prepacks)
+                dependencies = self.kit_dependencies.get(product_id, [])
+                is_standalone = len(dependencies) == 0
+                return "master", is_standalone
+            else:
+                return "unknown", False
+        else:
+            return "unclassified", False
+    
+    def is_product_ready_for_optimization(self, product_id: str) -> Tuple[bool, List[str]]:
+        """
+        Check if a product is ready for optimization.
+        
+        Returns:
+            Tuple[bool, List[str]]: (is_ready, exclusion_reasons)
+        """
+        exclusion_reasons = []
+        
+        # Classify product type
+        product_type, is_standalone_master = self.classify_product_type(product_id)
+        
+        # Check line assignment logic
+        has_line_assignment = product_id in self.line_assignments
+        
+        # For masters: standalone should have line assignment, non-standalone should NOT
+        if product_type == "master":
+            if is_standalone_master:
+                # Standalone masters should have "long line" assignment
+                if not has_line_assignment:
+                    exclusion_reasons.append("Standalone master missing line assignment")
+                elif self.line_assignments.get(product_id) != 6:  # 6 = LONG_LINE
+                    exclusion_reasons.append("Standalone master should have long line assignment")
+            else:
+                # Non-standalone masters should NOT have line assignment (excluded from production)
+                exclusion_reasons.append("Non-standalone master (excluded from production)")
+        else:
+            # For subkits and prepacks, check normal line assignment
+            if not has_line_assignment:
+                exclusion_reasons.append("No line assignment")
+        
+        # Check staffing requirements
+        unicef_staff = self.team_requirements.get('UNICEF Fixed term', {}).get(product_id, 0)
+        humanizer_staff = self.team_requirements.get('Humanizer', {}).get(product_id, 0)
+        total_staff = unicef_staff + humanizer_staff
+        
+        if total_staff == 0:
+            exclusion_reasons.append("Zero staffing requirements")
+        
+        is_ready = len(exclusion_reasons) == 0
+        return is_ready, exclusion_reasons
+    
+    def filter_products(self) -> Tuple[List[str], Dict[str, int], List[str], Dict[str, int]]:
+        """
+        Filter products into included and excluded lists.
+        
+        Returns:
+            Tuple containing:
+            - included_products: List of product IDs ready for optimization
+            - included_demand: Dict of {product_id: demand} for included products
+            - excluded_products: List of product IDs excluded from optimization
+            - excluded_demand: Dict of {product_id: demand} for excluded products
+        """
+        if not self.load_data():
+            raise Exception("Failed to load data for filtering")
+        
+        included_products = []
+        included_demand = {}
+        excluded_products = []
+        excluded_demand = {}
+        excluded_details = {}
+        
+        print("🔍 FILTERING DEMAND DATA FOR OPTIMIZATION")
+        
+        for product_id, demand in self.demand_data.items():
+            if demand <= 0:  # Skip products with no demand
+                continue
+                
+            is_ready, exclusion_reasons = self.is_product_ready_for_optimization(product_id)
+            
+            if is_ready:
+                included_products.append(product_id)
+                included_demand[product_id] = demand
+            else:
+                excluded_products.append(product_id)
+                excluded_demand[product_id] = demand
+                excluded_details[product_id] = exclusion_reasons
+        
+        # Sort products for consistent output
+        included_products.sort()
+        excluded_products.sort()
+        
+        # Print summary
+        total_demand = sum(self.demand_data.values())
+        included_total = sum(included_demand.values())
+        excluded_total = sum(excluded_demand.values())
+        
+        print(f"✅ INCLUDED in optimization: {len(included_products)} products ({included_total:,} units)")
+        print(f"🚫 EXCLUDED from optimization: {len(excluded_products)} products ({excluded_total:,} units)")
+        print(f"📊 Total demand: {total_demand:,} units")
+        
+        # Print exclusion breakdown
+        if excluded_products:
+            print(f"\n📋 EXCLUSION BREAKDOWN:")
+            reason_counts = {}
+            for reasons in excluded_details.values():
+                for reason in reasons:
+                    reason_counts[reason] = reason_counts.get(reason, 0) + 1
+            
+            for reason, count in reason_counts.items():
+                print(f"  • {reason}: {count} products")
+        
+        return included_products, included_demand, excluded_products, excluded_demand
+    
+    def get_filtered_product_list(self) -> List[str]:
+        """Get list of products ready for optimization"""
+        included_products, _, _, _ = self.filter_products()
+        return included_products
+    
+    def get_filtered_demand_dictionary(self) -> Dict[str, int]:
+        """Get demand dictionary for products ready for optimization"""
+        _, included_demand, _, _ = self.filter_products()
+        return included_demand
+    
+    def get_complete_product_analysis(self) -> Dict:
+        """Get complete analysis of all products for visualization"""
+        included_products, included_demand, excluded_products, excluded_demand = self.filter_products()
+        
+        all_products = {**included_demand, **excluded_demand}
+        product_details = {}
+        
+        for product_id, demand in all_products.items():
+            product_type, is_standalone_master = self.classify_product_type(product_id)
+            is_ready, exclusion_reasons = self.is_product_ready_for_optimization(product_id)
+            
+            # Get staffing info
+            unicef_staff = self.team_requirements.get('UNICEF Fixed term', {}).get(product_id, 0)
+            humanizer_staff = self.team_requirements.get('Humanizer', {}).get(product_id, 0)
+            
+            # Get line assignment
+            line_assignment = self.line_assignments.get(product_id)
+            
+            product_details[product_id] = {
+                'demand': demand,
+                'product_type': product_type,
+                'is_standalone_master': is_standalone_master,
+                'is_included_in_optimization': is_ready,
+                'exclusion_reasons': exclusion_reasons,
+                'unicef_staff': unicef_staff,
+                'humanizer_staff': humanizer_staff,
+                'total_staff': unicef_staff + humanizer_staff,
+                'line_assignment': line_assignment,
+                'has_line_assignment': line_assignment is not None,
+                'has_staffing': (unicef_staff + humanizer_staff) > 0,
+                'has_hierarchy': product_type != "unclassified"
+            }
+        
+        return {
+            'included_count': len(included_products),
+            'included_demand': sum(included_demand.values()),
+            'excluded_count': len(excluded_products),
+            'excluded_demand': sum(excluded_demand.values()),
+            'total_products': len(all_products),
+            'total_demand': sum(all_products.values()),
+            'product_details': product_details,
+            'standalone_masters_count': sum(1 for p in product_details.values() if p['is_standalone_master']),
+            'included_products': included_products,
+            'excluded_products': excluded_products
+        }
+    
+    def get_exclusion_summary(self) -> Dict:
+        """Get summary of excluded products for reporting"""
+        included_products, included_demand, excluded_products, excluded_demand = self.filter_products()
+        
+        excluded_details = {}
+        for product_id in excluded_products:
+            _, reasons = self.is_product_ready_for_optimization(product_id)
+            excluded_details[product_id] = {
+                'demand': excluded_demand[product_id],
+                'reasons': reasons
+            }
+        
+        return {
+            'included_count': len(included_products),
+            'included_demand': sum(included_demand.values()),
+            'excluded_count': len(excluded_products),
+            'excluded_demand': sum(excluded_demand.values()),
+            'excluded_details': excluded_details
+        }
+
+
+# Convenience functions for easy import
+def get_filtered_product_list() -> List[str]:
+    """Get list of products ready for optimization"""
+    filter_instance = DemandFilter()
+    return filter_instance.get_filtered_product_list()
+
+
+def get_filtered_demand_dictionary() -> Dict[str, int]:
+    """Get demand dictionary for products ready for optimization"""
+    filter_instance = DemandFilter()
+    return filter_instance.get_filtered_demand_dictionary()
+
+
+def get_exclusion_summary() -> Dict:
+    """Get summary of excluded products for reporting"""
+    filter_instance = DemandFilter()
+    return filter_instance.get_exclusion_summary()
+
+
+if __name__ == "__main__":
+    # Test the filtering
+    filter_instance = DemandFilter()
+    included_products, included_demand, excluded_products, excluded_demand = filter_instance.filter_products()
+    
+    print(f"\n=== FILTERING TEST RESULTS ===")
+    print(f"Included products: {included_products[:5]}..." if len(included_products) > 5 else f"Included products: {included_products}")
+    print(f"Excluded products: {excluded_products[:5]}..." if len(excluded_products) > 5 else f"Excluded products: {excluded_products}")

src/demand_validation_viz.py

@@ -0,0 +1,394 @@
+#!/usr/bin/env python3
+"""
+Demand Data Validation Visualization Module
+
+This module provides Streamlit visualization for demand data validation,
+showing which products are included/excluded from optimization and why.
+Uses the demand_filtering module for the actual filtering logic.
+"""
+
+import pandas as pd
+import streamlit as st
+from typing import Dict, List, Tuple, Optional
+import json
+from src.config.constants import ShiftType, LineType, KitLevel
+from src.demand_filtering import DemandFilter
+
+
+class DemandValidationViz:
+    """
+    Provides visualization for demand data validation using the filtering module
+    """
+    
+    def __init__(self):
+        self.filter_instance = DemandFilter()
+        self.speed_data = None
+        
+    def load_data(self):
+        """Load data needed for visualization"""
+        try:
+            # Load speed data for visualization
+            from src.config import optimization_config
+            self.speed_data = optimization_config.PER_PRODUCT_SPEED
+            
+            # Load data in the filter instance
+            return self.filter_instance.load_data()
+            
+        except Exception as e:
+            error_msg = f"Error loading data: {str(e)}"
+            print(error_msg)
+            try:
+                st.error(error_msg)
+            except:
+                pass
+            return False
+    
+    def classify_product_type(self, product_id: str) -> Tuple[str, str]:
+        """Get product classification from filter instance"""
+        if product_id in self.filter_instance.kit_levels:
+            level = self.filter_instance.kit_levels[product_id]
+            level_name = KitLevel.get_name(level)
+            
+            if level == 0:
+                return "prepack", level_name
+            elif level == 1:
+                return "subkit", level_name
+            elif level == 2:
+                return "master", level_name
+            else:
+                return "unknown", f"level_{level}"
+        else:
+            return "unclassified", "no_hierarchy_data"
+    
+    def get_line_assignment(self, product_id: str) -> Tuple[Optional[int], str]:
+        """Get line assignment from filter instance"""
+        if product_id in self.filter_instance.line_assignments:
+            line_type = self.filter_instance.line_assignments[product_id]
+            line_name = LineType.get_name(line_type)
+            return line_type, line_name
+        else:
+            return None, "no_assignment"
+    
+    def get_staffing_requirements(self, product_id: str) -> Dict[str, int]:
+        """Get staffing requirements from filter instance"""
+        staffing = {}
+        
+        for emp_type, products in self.filter_instance.team_requirements.items():
+            staffing[emp_type] = products.get(product_id, 0)
+        
+        return staffing
+    
+    def get_production_speed(self, product_id: str) -> Optional[float]:
+        """Get production speed for product"""
+        return self.speed_data.get(product_id, None)
+    
+    def validate_all_products(self) -> pd.DataFrame:
+        """
+        Create visualization dataframe using complete analysis from filtering module
+        Returns: DataFrame with validation results for display
+        """
+        # Get complete analysis from filtering module
+        analysis = self.filter_instance.get_complete_product_analysis()
+        product_details = analysis['product_details']
+        
+        results = []
+        
+        for product_id, details in product_details.items():
+            # Get additional data for visualization
+            speed = self.get_production_speed(product_id)
+            production_hours_needed = None
+            if speed and speed > 0:
+                production_hours_needed = details['demand'] / speed
+            
+            # Get line type name
+            line_type_id = details['line_assignment']
+            line_name = "no_assignment"
+            if line_type_id is not None:
+                from src.config.constants import LineType
+                line_name = LineType.get_name(line_type_id)
+            
+            # Get level name
+            level_name = "no_hierarchy_data"
+            if details['product_type'] != "unclassified":
+                if details['product_type'] == "prepack":
+                    level_name = "prepack"
+                elif details['product_type'] == "subkit":
+                    level_name = "subkit"
+                elif details['product_type'] == "master":
+                    if details['is_standalone_master']:
+                        level_name = "standalone_master"
+                    else:
+                        level_name = "master_with_hierarchy"
+                else:
+                    level_name = f"level_{details['product_type']}"
+            
+            # Overall status
+            if not details['is_included_in_optimization']:
+                validation_status = f"🚫 Excluded: {', '.join(details['exclusion_reasons'])}"
+            else:
+                # Check for other issues that don't exclude from optimization
+                other_issues = []
+                if speed is None:
+                    other_issues.append("no_speed_data")
+                if not details['has_hierarchy']:
+                    other_issues.append("no_hierarchy_data")
+                
+                if other_issues:
+                    validation_status = f"⚠️ Issues: {', '.join(other_issues)}"
+                else:
+                    validation_status = "✅ Included in optimization"
+            
+            results.append({
+                'Product ID': product_id,
+                'Demand': details['demand'],
+                'Product Type': details['product_type'].title(),
+                'Level': level_name,
+                'Is Standalone Master': "Yes" if details['is_standalone_master'] else "No",
+                'Line Type ID': line_type_id if line_type_id else "N/A",
+                'Line Type': line_name,
+                'UNICEF Staff': details['unicef_staff'],
+                'Humanizer Staff': details['humanizer_staff'],
+                'Total Staff': details['total_staff'],
+                'Production Speed (units/hour)': f"{speed:.1f}" if speed else "N/A",
+                'Production Hours Needed': f"{production_hours_needed:.1f}" if production_hours_needed else "N/A",
+                'Has Line Assignment': "✅" if details['has_line_assignment'] else "❌",
+                'Has Staffing Data': "✅" if details['has_staffing'] else "❌", 
+                'Has Speed Data': "✅" if speed is not None else "❌",
+                'Has Hierarchy Data': "✅" if details['has_hierarchy'] else "❌",
+                'Excluded from Optimization': not details['is_included_in_optimization'],
+                'Exclusion Reasons': ', '.join(details['exclusion_reasons']) if details['exclusion_reasons'] else '',
+                'Validation Status': validation_status
+            })
+        
+        df = pd.DataFrame(results)
+        
+        # Sort by exclusion status first, then by demand
+        df = df.sort_values(['Excluded from Optimization', 'Demand'], ascending=[False, False])
+        
+        return df
+    
+    def get_summary_statistics(self, df: pd.DataFrame) -> Dict:
+        """Generate summary statistics using filtering module analysis"""
+        
+        # Get analysis from filtering module
+        analysis = self.filter_instance.get_complete_product_analysis()
+        
+        # Calculate issues for included products only
+        included_df = df[df['Excluded from Optimization'] == False]
+        
+        no_line_assignment = len(included_df[included_df['Has Line Assignment'] == "❌"])
+        no_staffing = len(included_df[included_df['Has Staffing Data'] == "❌"])
+        no_speed = len(included_df[included_df['Has Speed Data'] == "❌"])
+        no_hierarchy = len(included_df[included_df['Has Hierarchy Data'] == "❌"])
+        
+        # Product type and line type distributions
+        type_counts = df['Product Type'].value_counts().to_dict()
+        
+        # Staffing summary from analysis
+        total_unicef_needed = sum(p['unicef_staff'] for p in analysis['product_details'].values())
+        total_humanizer_needed = sum(p['humanizer_staff'] for p in analysis['product_details'].values())
+        
+        return {
+            'total_products': analysis['total_products'],
+            'total_demand': analysis['total_demand'],
+            'included_products': analysis['included_count'],
+            'excluded_products': analysis['excluded_count'],
+            'included_demand': analysis['included_demand'],
+            'excluded_demand': analysis['excluded_demand'],
+            'type_counts': type_counts,
+            'no_line_assignment': no_line_assignment,
+            'no_staffing': no_staffing,
+            'no_speed': no_speed,
+            'no_hierarchy': no_hierarchy,
+            'standalone_masters': analysis['standalone_masters_count'],
+            'total_unicef_needed': total_unicef_needed,
+            'total_humanizer_needed': total_humanizer_needed
+        }
+
+
+def display_demand_validation():
+    """
+    Display demand validation analysis in Streamlit
+    """
+    st.header("📋 Demand Data Validation")
+    st.markdown("Comprehensive analysis of products with demand to identify potential optimization issues.")
+    
+    # Initialize validator
+    validator = DemandValidationViz()
+    
+    # Load data
+    with st.spinner("Loading data for validation..."):
+        if not validator.load_data():
+            st.error("Failed to load data for validation.")
+            return
+    
+    # Perform validation
+    with st.spinner("Analyzing demand data..."):
+        validation_df = validator.validate_all_products()
+        summary_stats = validator.get_summary_statistics(validation_df)
+    
+    # Display summary statistics
+    st.subheader("📊 Summary Statistics")
+    
+    col1, col2, col3, col4 = st.columns(4)
+    
+    with col1:
+        st.metric("Total Products", summary_stats['total_products'])
+        st.metric("Included in Optimization", summary_stats['included_products'], delta="Ready for optimization")
+    
+    with col2:
+        st.metric("Total Demand", f"{summary_stats['total_demand']:,}")
+        st.metric("Excluded from Optimization", summary_stats['excluded_products'], delta="Omitted")
+    
+    with col3:
+        st.metric("Included Demand", f"{summary_stats['included_demand']:,}", delta="Will be optimized")
+        st.metric("UNICEF Staff Needed", summary_stats['total_unicef_needed'])
+    
+    with col4:
+        st.metric("Excluded Demand", f"{summary_stats['excluded_demand']:,}", delta="Omitted")
+        st.metric("Humanizer Staff Needed", summary_stats['total_humanizer_needed'])
+    
+    # Product type distribution
+    st.subheader("📈 Product Type Distribution")
+    if summary_stats['type_counts']:
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            type_df = pd.DataFrame(list(summary_stats['type_counts'].items()), 
+                                 columns=['Product Type', 'Count'])
+            st.bar_chart(type_df.set_index('Product Type'))
+        
+        with col2:
+            for ptype, count in summary_stats['type_counts'].items():
+                percentage = (count / summary_stats['total_products']) * 100
+                st.write(f"**{ptype}:** {count} products ({percentage:.1f}%)")
+    
+    # Validation issues summary for included products
+    st.subheader("⚠️ Data Quality Issues (Products Included in Optimization)")
+    st.write("Issues affecting products that **will be** included in optimization:")
+    
+    col1, col2, col3, col4 = st.columns(4)
+    
+    with col1:
+        st.metric("No Line Assignment", summary_stats['no_line_assignment'], 
+                 delta=None if summary_stats['no_line_assignment'] == 0 else "Issue")
+    
+    with col2:
+        st.metric("No Staffing Data", summary_stats['no_staffing'],
+                 delta=None if summary_stats['no_staffing'] == 0 else "Issue")
+    
+    with col3:
+        st.metric("No Speed Data", summary_stats['no_speed'],
+                 delta=None if summary_stats['no_speed'] == 0 else "Issue")
+    
+    with col4:
+        st.metric("No Hierarchy Data", summary_stats['no_hierarchy'],
+                 delta=None if summary_stats['no_hierarchy'] == 0 else "Issue")
+    
+    # Separate the results into included and excluded
+    included_df = validation_df[validation_df['Excluded from Optimization'] == False].copy()
+    excluded_df = validation_df[validation_df['Excluded from Optimization'] == True].copy()
+    
+    # Products Included in Optimization
+    st.subheader("✅ Products Included in Optimization")
+    st.write(f"**{len(included_df)} products** will be included in the optimization with total demand of **{included_df['Demand'].sum():,} units**")
+    
+    if len(included_df) > 0:
+        # Filter options for included products
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            included_type_filter = st.selectbox("Filter included by type", 
+                                              options=["All"] + list(included_df['Product Type'].unique()),
+                                              key="included_filter")
+        
+        with col2:
+            included_min_demand = st.number_input("Minimum demand (included)", min_value=0, value=0, key="included_demand")
+        
+        # Apply filters to included
+        filtered_included = included_df.copy()
+        if included_type_filter != "All":
+            filtered_included = filtered_included[filtered_included['Product Type'] == included_type_filter]
+        if included_min_demand > 0:
+            filtered_included = filtered_included[filtered_included['Demand'] >= included_min_demand]
+        
+        # Configure column display for included
+        included_columns = ['Product ID', 'Demand', 'Product Type', 'Line Type', 'UNICEF Staff', 'Humanizer Staff', 'Production Speed (units/hour)', 'Validation Status']
+        
+        st.dataframe(
+            filtered_included[included_columns],
+            use_container_width=True,
+            height=300
+        )
+    else:
+        st.warning("No products are included in optimization!")
+    
+    # Products Excluded from Optimization  
+    st.subheader("🚫 Products Excluded from Optimization")
+    st.write(f"**{len(excluded_df)} products** are excluded from optimization with total demand of **{excluded_df['Demand'].sum():,} units**")
+    st.info("These products are omitted from optimization due to missing line assignments or zero staffing requirements.")
+    
+    if len(excluded_df) > 0:
+        # Show exclusion breakdown
+        exclusion_reasons = excluded_df['Exclusion Reasons'].value_counts()
+        st.write("**Exclusion reasons:**")
+        for reason, count in exclusion_reasons.items():
+            st.write(f"• {reason}: {count} products")
+        
+        # Configure column display for excluded
+        excluded_columns = ['Product ID', 'Demand', 'Product Type', 'Exclusion Reasons', 'UNICEF Staff', 'Humanizer Staff', 'Line Type']
+        
+        st.dataframe(
+            excluded_df[excluded_columns],
+            use_container_width=True,
+            height=200
+        )
+        
+        # Export option
+        if st.button("📥 Export Validation Results to CSV"):
+            csv = validation_df.to_csv(index=False)
+            st.download_button(
+                label="Download CSV",
+                data=csv,
+                file_name="demand_validation_results.csv",
+                mime="text/csv"
+            )
+    else:
+        st.info("No products match the selected filters.")
+    
+    # Recommendations
+    st.subheader("💡 Recommendations")
+    
+    recommendations = []
+    
+    # Focus on exclusion criteria first
+    if summary_stats['excluded_products'] > 0:
+        st.warning(f"**Optimization Scope**: {summary_stats['excluded_products']} products ({summary_stats['excluded_demand']:,} units demand) are excluded from optimization.")
+    
+    # Data quality issues for INCLUDED products only
+    if summary_stats['no_line_assignment'] > 0:
+        recommendations.append(f"**Line Assignment**: {summary_stats['no_line_assignment']} products included in optimization are missing line assignments.")
+    
+    if summary_stats['no_staffing'] > 0:
+        recommendations.append(f"**Staffing Data**: {summary_stats['no_staffing']} products included in optimization are missing staffing requirements.")
+    
+    if summary_stats['no_speed'] > 0:
+        recommendations.append(f"**Speed Data**: {summary_stats['no_speed']} products included in optimization are missing production speed data (will use defaults).")
+    
+    if summary_stats['no_hierarchy'] > 0:
+        recommendations.append(f"**Hierarchy Data**: {summary_stats['no_hierarchy']} products included in optimization are not in the kit hierarchy.")
+    
+    if recommendations:
+        for rec in recommendations:
+            st.info(rec)
+    
+    # Overall status
+    if summary_stats['included_products'] > 0:
+        st.success(f"✅ **Ready for Optimization**: {summary_stats['included_products']} products with {summary_stats['included_demand']:,} units demand are ready for optimization!")
+    else:
+        st.error("❌ No products are ready for optimization. Please review the exclusion criteria above.")
+
+
+if __name__ == "__main__":
+    # For testing
+    display_demand_validation()

src/etl/extract.py

@@ -179,11 +179,17 @@ def get_production_order_data():
             kit_levels[subkit_id] = 1
             dependencies[subkit_id] = subkit_data.get('dependencies', [])
             
-            # Process prepacks (level 0)
+            # Process prepacks under subkits (level 0)
             for prepack_id in subkit_data.get('prepacks', []):
                 if prepack_id not in kit_levels:  # Avoid overwriting if already exists
                     kit_levels[prepack_id] = 0
                     dependencies[prepack_id] = []
+        
+        # Process direct prepacks under master (level 0)
+        for prepack_id in master_data.get('direct_prepacks', []):
+            if prepack_id not in kit_levels:  # Avoid overwriting if already exists
+                kit_levels[prepack_id] = 0
+                dependencies[prepack_id] = []
     
     # Create priority order: prepacks first, then subkits, then masters
     priority_order = []

src/etl/hierarchy_parser.py

@@ -86,6 +86,20 @@ class KitHierarchyParser:
                         # Add prepack to subkit dependencies
                         if prepack_id not in hierarchy[master_id]['subkits'][subkit_id]['dependencies']:
                             hierarchy[master_id]['subkits'][subkit_id]['dependencies'].append(prepack_id)
+                            
+                elif pd.notna(prepack_id):
+                    # Handle direct master-prepack relationship (no subkit)
+                    # Add direct_prepacks list to hierarchy if it doesn't exist
+                    if 'direct_prepacks' not in hierarchy[master_id]:
+                        hierarchy[master_id]['direct_prepacks'] = []
+                    
+                    # Add prepack directly to master
+                    if prepack_id not in hierarchy[master_id]['direct_prepacks']:
+                        hierarchy[master_id]['direct_prepacks'].append(prepack_id)
+                    
+                    # Add prepack to master dependencies
+                    if prepack_id not in hierarchy[master_id]['dependencies']:
+                        hierarchy[master_id]['dependencies'].append(prepack_id)
         
         # Convert defaultdict to regular dict for JSON serialization
         self.hierarchy_json = json.loads(json.dumps(hierarchy, default=dict))

src/utils/excel_to_csv_converter.py

@@ -35,7 +35,6 @@ def analyze_excel_structure(excel_path):
             print(f"   - Rows: {len(df)}")
             print(f"   - Columns: {len(df.columns)}")
             print(f"   - Column names: {list(df.columns)}")
-            print()
         
         return sheet_info
         
@@ -97,7 +96,7 @@ def main():
     
     # Define paths
     excel_path = "data/real_data_excel/AI Project document.xlsx"
-    output_dir = "data/converted_csv"
+    output_dir = "data/real_data_excel/converted_csv"
     
     # Check if Excel file exists
     if not os.path.exists(excel_path):

src/utils/kit_composition_cleaner.py

@@ -35,17 +35,25 @@ def load_kit_composition_data(file_path: str) -> pd.DataFrame:
 def process_master_kits(df: pd.DataFrame) -> pd.DataFrame:
     """
     Process Master Kits according to business rules:
-    - Standalone masters (appear only once): line_type = "long line"
-    - Non-standalone masters: line_type = "" (empty)
+    - Standalone masters (no subkits/prepacks, only components): line_type = "long line"
+    - Non-standalone masters (have subkits/prepacks): line_type = "" (empty - no production needed)
     """
     print("Processing Master Kits...")
     
-    # Get master kit counts to identify standalone masters
-    master_counts = df['Master Kit'].value_counts()
-    standalone_masters = set(master_counts[master_counts == 1].index)
+    # Identify masters with hierarchy (subkits or prepacks)
+    masters_with_subkits = set(df[df['Sub kit'].notna()]['Master Kit'].unique())
+    masters_with_prepacks = set(df[df['Prepack'].notna()]['Master Kit'].unique())
+    masters_with_hierarchy = masters_with_subkits.union(masters_with_prepacks)
     
-    print(f"Total unique Master Kits: {len(master_counts)}")
-    print(f"Standalone masters (appear only once): {len(standalone_masters)}")
+    # All masters
+    all_masters = set(df['Master Kit'].unique())
+    
+    # Standalone masters are those WITHOUT subkits/prepacks (only have components)
+    standalone_masters = all_masters - masters_with_hierarchy
+    
+    print(f"Total unique Master Kits: {len(all_masters)}")
+    print(f"Masters with subkits/prepacks: {len(masters_with_hierarchy)}")
+    print(f"Standalone masters (only components): {len(standalone_masters)}")
     
     # Create master kit records
     master_data = []