Update sozo_gen.py

sozo_gen.py

@@ -268,388 +268,352 @@ def sanitize_for_firebase_key(text: str) -> str:
     return text
 
 # REPLACE THE OLD generate_report_draft WITH THIS CORRECTED VERSION
-def generate_report_draft(buf, name: str, ctx: str, uid: str, project_id: str, bucket):
+from scipy import stats
+import re
+
+def analyze_data_intelligence(df: pd.DataFrame, ctx_dict: Dict) -> Dict[str, Any]:
     """
-    Enhanced autonomous data analysis function that intelligently analyzes any dataset
-    and generates comprehensive, domain-appropriate reports with contextual visualizations.
-    
-    Maintains backward compatibility with existing function signature and outputs.
+    Autonomous data intelligence system that classifies domain, 
+    detects patterns, and determines optimal analytical approach.
     """
-    logging.info(f"Generating enhanced autonomous report draft for project {project_id}")
     
-    # Load data safely (existing functionality preserved)
-    df = load_dataframe_safely(buf, name)
+    # Domain Classification Engine
+    domain_signals = {
+        'financial': ['amount', 'price', 'cost', 'revenue', 'profit', 'balance', 'transaction', 'payment'],
+        'survey': ['rating', 'satisfaction', 'score', 'response', 'feedback', 'opinion', 'agree', 'likert'],
+        'scientific': ['measurement', 'experiment', 'trial', 'test', 'control', 'variable', 'hypothesis'],
+        'marketing': ['campaign', 'conversion', 'click', 'impression', 'engagement', 'customer', 'segment'],
+        'operational': ['performance', 'efficiency', 'throughput', 'capacity', 'utilization', 'process'],
+        'temporal': ['date', 'time', 'timestamp', 'period', 'month', 'year', 'day', 'hour']
+    }
     
-    # Initialize LLM (existing setup preserved)
-    llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", google_api_key=API_KEY, temperature=0.1)
+    # Analyze column patterns
+    columns_lower = [col.lower() for col in df.columns]
+    domain_scores = {}
     
-    # Enhanced autonomous data analysis
-    try:
-        # Stage 1: Intelligent Data Classification and Deep Analysis
-        autonomous_context = perform_autonomous_data_analysis(df, ctx, name)
-        
-        # Stage 2: Generate Enhanced Report with Intelligent Narrative
-        enhanced_report = generate_intelligent_report(llm, autonomous_context)
-        
-        # Stage 3: Smart Chart Generation
-        chart_urls = generate_autonomous_charts(llm, df, enhanced_report, uid, project_id, bucket)
-        
-        # Preserve original output structure
-        return {"raw_md": enhanced_report, "chartUrls": chart_urls}
-        
-    except Exception as e:
-        logging.error(f"Enhanced analysis failed, falling back to original: {str(e)}")
-        # Fallback to original logic if enhancement fails
-        return generate_original_report(df, llm, ctx, uid, project_id, bucket)
-
-
-def perform_autonomous_data_analysis(df: pd.DataFrame, user_ctx: str, filename: str) -> Dict[str, Any]:
-    """
-    Performs comprehensive autonomous analysis of the dataset to understand its nature,
-    domain, and analytical potential.
-    """
-    logging.info("Performing autonomous data analysis...")
-    
-    # Basic data profiling with JSON-safe types
-    basic_info = {
-        "shape": df.shape,
-        "columns": list(df.columns),
-        "dtypes": {col: str(dtype) for col, dtype in df.dtypes.items()},
-        "filename": filename,
-        "user_context": user_ctx
-    }
+    for domain, keywords in domain_signals.items():
+        score = sum(1 for col in columns_lower if any(keyword in col for keyword in keywords))
+        domain_scores[domain] = score
     
-    # Intelligent domain classification
-    domain_analysis = classify_dataset_domain(df, filename)
+    # Determine primary domain
+    primary_domain = max(domain_scores, key=domain_scores.get) if max(domain_scores.values()) > 0 else 'general'
+    
+    # Data Structure Analysis
+    numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    categorical_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()
+    datetime_cols = df.select_dtypes(include=['datetime64']).columns.tolist()
+    
+    # Detect time series
+    is_timeseries = len(datetime_cols) > 0 or any('date' in col.lower() or 'time' in col.lower() for col in columns_lower)
+    
+    # Statistical Profile
+    statistical_summary = {}
+    if numeric_cols:
+        statistical_summary = {
+            'correlations': df[numeric_cols].corr().abs().max().to_dict(),
+            'distributions': {col: 'normal' if stats.normaltest(df[col].dropna())[1] > 0.05 else 'non_normal' 
+                           for col in numeric_cols if len(df[col].dropna()) > 8},
+            'outliers': {col: len(df[col][np.abs(stats.zscore(df[col].dropna())) > 3]) 
+                        for col in numeric_cols if len(df[col].dropna()) > 0}
+        }
     
-    # Advanced statistical analysis
-    statistical_profile = generate_statistical_profile(df)
+    # Pattern Detection
+    patterns = {
+        'has_missing_data': df.isnull().sum().sum() > 0,
+        'has_duplicates': df.duplicated().sum() > 0,
+        'has_negative_values': any(df[col].min() < 0 for col in numeric_cols if len(df[col].dropna()) > 0),
+        'has_categorical_hierarchy': any(len(df[col].unique()) > 10 for col in categorical_cols),
+        'potential_segments': len(categorical_cols) > 0
+    }
+    
+    # Insight Opportunities
+    insight_opportunities = []
     
-    # Relationship discovery
-    relationships = discover_data_relationships(df)
+    if is_timeseries:
+        insight_opportunities.append("temporal_trends")
     
-    # Temporal analysis if applicable
-    temporal_insights = analyze_temporal_patterns(df)
+    if len(numeric_cols) > 1:
+        insight_opportunities.append("correlations")
     
-    # Data quality assessment
-    quality_metrics = assess_data_quality(df)
+    if len(categorical_cols) > 0 and len(numeric_cols) > 0:
+        insight_opportunities.append("segmentation")
     
-    # Business context inference
-    business_context = infer_business_context(df, domain_analysis)
+    if any(statistical_summary.get('outliers', {}).values()):
+        insight_opportunities.append("anomalies")
     
     return {
-        "basic_info": basic_info,
-        "domain": domain_analysis,
-        "statistics": statistical_profile,
-        "relationships": relationships,
-        "temporal": temporal_insights,
-        "quality": quality_metrics,
-        "business_context": business_context,
-        "analysis_complexity": determine_analysis_complexity(df, domain_analysis)
+        'primary_domain': primary_domain,
+        'domain_confidence': domain_scores,
+        'data_structure': {
+            'is_timeseries': is_timeseries,
+            'numeric_cols': numeric_cols,
+            'categorical_cols': categorical_cols,
+            'datetime_cols': datetime_cols
+        },
+        'statistical_profile': statistical_summary,
+        'patterns': patterns,
+        'insight_opportunities': insight_opportunities,
+        'narrative_suggestions': get_narrative_suggestions(primary_domain, insight_opportunities, patterns)
     }
 
+def get_narrative_suggestions(domain: str, opportunities: List[str], patterns: Dict) -> Dict[str, str]:
+    """Generate narrative direction based on domain and data characteristics"""
+    
+    narrative_frameworks = {
+        'financial': {
+            'hook': "Follow the money trail that reveals your business's hidden opportunities",
+            'structure': "performance → trends → risks → opportunities",
+            'focus': "profitability, efficiency, growth patterns, risk indicators"
+        },
+        'survey': {
+            'hook': "Your customers are speaking - here's what they're really saying",
+            'structure': "sentiment → segments → drivers → actions",
+            'focus': "satisfaction drivers, demographic patterns, improvement areas"
+        },
+        'scientific': {
+            'hook': "The data reveals relationships that challenge conventional thinking",
+            'structure': "hypothesis → evidence → significance → implications",
+            'focus': "statistical significance, correlations, experimental validity"
+        },
+        'marketing': {
+            'hook': "Discover the customer journey patterns driving your growth",
+            'structure': "performance → segments → optimization → strategy",
+            'focus': "conversion funnels, customer segments, campaign effectiveness"
+        },
+        'operational': {
+            'hook': "Operational excellence lives in the details - here's where to look",
+            'structure': "efficiency → bottlenecks → optimization → impact",
+            'focus': "process efficiency, capacity utilization, improvement opportunities"
+        },
+        'general': {
+            'hook': "Every dataset tells a story - here's what yours is saying",
+            'structure': "overview → patterns → insights → implications",
+            'focus': "key patterns, significant relationships, actionable insights"
+        }
+    }
+    
+    return narrative_frameworks.get(domain, narrative_frameworks['general'])
 
-def classify_dataset_domain(df: pd.DataFrame, filename: str) -> Dict[str, Any]:
+def create_autonomous_prompt(df: pd.DataFrame, enhanced_ctx: Dict, intelligence: Dict) -> str:
     """
-    Intelligently classifies the dataset domain based on column patterns, data types,
-    and semantic analysis.
+    Generate a dynamic, intelligence-driven prompt that creates compelling narratives
+    rather than following templates.
     """
-    domain_indicators = {
-        "financial": ["amount", "price", "cost", "revenue", "profit", "transaction", "payment", "invoice"],
-        "survey": ["rating", "satisfaction", "response", "score", "survey", "feedback", "opinion"],
-        "scientific": ["measurement", "experiment", "test", "sample", "observation", "hypothesis", "variable"],
-        "marketing": ["campaign", "click", "conversion", "customer", "lead", "acquisition", "retention"],
-        "operational": ["process", "time", "duration", "status", "workflow", "performance", "efficiency"],
-        "sales": ["order", "product", "quantity", "sales", "customer", "deal", "pipeline"],
-        "hr": ["employee", "salary", "department", "performance", "training", "recruitment"],
-        "healthcare": ["patient", "diagnosis", "treatment", "medical", "health", "symptom", "medication"]
-    }
     
-    # Analyze column names for domain indicators
-    columns_lower = [col.lower() for col in df.columns]
-    domain_scores = {}
-    
-    for domain, keywords in domain_indicators.items():
-        score = sum(1 for col in columns_lower for keyword in keywords if keyword in col)
-        domain_scores[domain] = score
+    domain = intelligence['primary_domain']
+    opportunities = intelligence['insight_opportunities']
+    narrative = intelligence['narrative_suggestions']
     
-    # Filename analysis
-    filename_lower = filename.lower()
-    for domain, keywords in domain_indicators.items():
-        if any(keyword in filename_lower for keyword in keywords):
-            domain_scores[domain] = domain_scores.get(domain, 0) + 2
+    # Dynamic chart strategy based on data characteristics
+    chart_strategy = generate_chart_strategy(intelligence)
     
-    # Data type analysis
-    numeric_ratio = len(df.select_dtypes(include=[np.number]).columns) / len(df.columns)
-    categorical_ratio = len(df.select_dtypes(include=['object']).columns) / len(df.columns)
-    
-    # Determine primary domain
-    primary_domain = max(domain_scores, key=domain_scores.get) if domain_scores else "general"
-    
-    return {
-        "primary_domain": primary_domain,
-        "domain_confidence": int(domain_scores.get(primary_domain, 0)),
-        "domain_scores": {k: int(v) for k, v in domain_scores.items()},
-        "data_characteristics": {
-            "numeric_ratio": float(numeric_ratio),
-            "categorical_ratio": float(categorical_ratio),
-            "is_time_series": detect_time_series(df),
-            "is_transactional": detect_transactional_data(df),
-            "is_experimental": detect_experimental_data(df)
-        }
-    }
+    prompt = f"""You are an elite data storyteller with deep expertise in {domain} analytics. Your mission is to uncover the compelling narrative hidden in this dataset and present it as a captivating story that drives action.
 
+**THE DATA'S STORY CONTEXT:**
+{json.dumps(enhanced_ctx, indent=2)}
 
-def generate_statistical_profile(df: pd.DataFrame) -> Dict[str, Any]:
-    """
-    Generates comprehensive statistical profile of the dataset.
-    """
-    profile = {
-        "summary_stats": {},
-        "correlations": {},
-        "distributions": {},
-        "outliers": {},
-        "missing_data": {}
+**INTELLIGENCE ANALYSIS:**
+- Primary Domain: {domain}
+- Key Opportunities: {', '.join(opportunities)}
+- Data Characteristics: {intelligence['data_structure']}
+- Narrative Framework: {narrative['structure']}
+
+**YOUR STORYTELLING MISSION:**
+{narrative['hook']}
+
+**NARRATIVE CONSTRUCTION GUIDELINES:**
+1. **LEAD WITH INTRIGUE**: Start with the most compelling finding that hooks the reader
+2. **BUILD TENSION**: Present contrasts, surprises, or unexpected patterns
+3. **REVEAL INSIGHTS**: Use data to resolve the tension with clear explanations
+4. **DRIVE ACTION**: End with specific, actionable recommendations
+
+**VISUALIZATION STRATEGY:**
+{chart_strategy}
+
+**CRITICAL INSTRUCTIONS:**
+- Write as if you're revealing a detective story, not filling a template
+- Every insight must be supported by data evidence
+- Use compelling headers that create curiosity (not "Executive Summary")
+- Weave charts naturally into the narrative flow
+- Focus on business impact and actionable outcomes
+- Let the data's personality shine through your writing style
+
+**CHART INTEGRATION:**
+Insert charts using: `<generate_chart: "chart_type | compelling description that advances the story">`
+Available types: bar, pie, line, scatter, hist
+
+Transform this data into a story that decision-makers can't stop reading."""
+
+    return prompt
+
+def generate_chart_strategy(intelligence: Dict) -> str:
+    """Generate visualization strategy based on data intelligence"""
+    
+    domain = intelligence['primary_domain']
+    opportunities = intelligence['insight_opportunities']
+    structure = intelligence['data_structure']
+    
+    strategies = {
+        'financial': "Focus on trend lines showing performance over time, comparative bars for different categories, and scatter plots revealing correlations between financial metrics.",
+        'survey': "Emphasize distribution histograms for satisfaction scores, segmented bar charts for demographic breakdowns, and correlation matrices for response patterns.",
+        'scientific': "Prioritize scatter plots with regression lines, distribution comparisons, and statistical significance visualizations.",
+        'marketing': "Highlight conversion funnels, customer segment comparisons, and campaign performance trends.",
+        'operational': "Show efficiency trends, capacity utilization charts, and process performance comparisons."
     }
     
-    # Summary statistics for numeric columns with JSON-safe conversion
-    numeric_cols = df.select_dtypes(include=[np.number]).columns
-    if len(numeric_cols) > 0:
-        desc_stats = df[numeric_cols].describe()
-        # Convert to JSON-safe format
-        profile["summary_stats"] = {
-            col: {
-                stat: float(val) if pd.notna(val) else None 
-                for stat, val in desc_stats[col].items()
-            }
-            for col in desc_stats.columns
-        }
-        
-        # Correlation analysis with JSON-safe conversion
-        if len(numeric_cols) > 1:
-            corr_matrix = df[numeric_cols].corr()
-            # Find strong correlations
-            strong_corrs = []
-            for i in range(len(corr_matrix.columns)):
-                for j in range(i+1, len(corr_matrix.columns)):
-                    corr_val = corr_matrix.iloc[i, j]
-                    if abs(corr_val) > 0.7 and pd.notna(corr_val):  # Strong correlation threshold
-                        strong_corrs.append({
-                            "var1": corr_matrix.columns[i],
-                            "var2": corr_matrix.columns[j],
-                            "correlation": float(corr_val)
-                        })
-            profile["correlations"] = {"strong_correlations": strong_corrs}
-    
-    # Categorical analysis
-    categorical_cols = df.select_dtypes(include=['object']).columns
-    if len(categorical_cols) > 0:
-        profile["categorical_analysis"] = {}
-        for col in categorical_cols:
-            value_counts = df[col].value_counts().head(5)
-            profile["categorical_analysis"][col] = {
-                "unique_count": int(df[col].nunique()),
-                "top_values": {str(k): int(v) for k, v in value_counts.items()}
-            }
+    base_strategy = strategies.get(domain, "Create visualizations that best tell your data's unique story.")
     
-    # Missing data analysis with JSON-safe conversion
-    missing_data = df.isnull().sum()
-    missing_dict = {}
-    for col, missing_count in missing_data.items():
-        if missing_count > 0:
-            missing_dict[col] = int(missing_count)
+    # Add specific guidance based on data characteristics
+    if structure['is_timeseries']:
+        base_strategy += " Leverage time-series visualizations to show trends and patterns over time."
     
-    profile["missing_data"] = {
-        "columns_with_missing": missing_dict,
-        "total_missing_percentage": float((df.isnull().sum().sum() / (len(df) * len(df.columns))) * 100)
-    }
+    if 'correlations' in opportunities:
+        base_strategy += " Include correlation visualizations to reveal hidden relationships."
     
-    return profile
-
+    if 'segmentation' in opportunities:
+        base_strategy += " Use segmented charts to highlight different groups or categories."
+    
+    return base_strategy
 
-def discover_data_relationships(df: pd.DataFrame) -> Dict[str, Any]:
-    """
-    Discovers meaningful relationships and patterns in the data.
-    """
-    relationships = {
-        "key_relationships": [],
-        "patterns": [],
-        "anomalies": []
-    }
+def enhance_data_context(df: pd.DataFrame, ctx_dict: Dict) -> Dict[str, Any]:
+    """Enhanced context generation with AI-driven analysis"""
     
-    # Identify potential key relationships
-    numeric_cols = df.select_dtypes(include=[np.number]).columns
+    # Get autonomous intelligence analysis
+    intelligence = analyze_data_intelligence(df, ctx_dict)
     
-    if len(numeric_cols) > 1:
-        # Find interesting relationships
-        for col1 in numeric_cols:
-            for col2 in numeric_cols:
-                if col1 != col2:
-                    correlation = df[col1].corr(df[col2])
-                    if abs(correlation) > 0.5 and pd.notna(correlation):  # Moderate to strong correlation
-                        relationships["key_relationships"].append({
-                            "variable1": col1,
-                            "variable2": col2,
-                            "relationship_strength": float(correlation),
-                            "relationship_type": "positive" if correlation > 0 else "negative"
-                        })
-    
-    # Identify patterns in categorical data
-    categorical_cols = df.select_dtypes(include=['object']).columns
-    for col in categorical_cols:
-        if df[col].nunique() < 20:  # Reasonable number of categories
-            value_counts = df[col].value_counts()
-            if len(value_counts) > 0:
-                relationships["patterns"].append({
-                    "column": col,
-                    "pattern_type": "categorical_distribution",
-                    "dominant_category": str(value_counts.index[0]),
-                    "dominance_percentage": float((value_counts.iloc[0] / len(df)) * 100)
-                })
+    # Original context enhancement
+    enhanced = ctx_dict.copy()
     
-    return relationships
-
-
-def analyze_temporal_patterns(df: pd.DataFrame) -> Dict[str, Any]:
-    """
-    Analyzes temporal patterns if time-based columns are detected.
-    """
-    temporal_insights = {"has_temporal_data": False}
+    # Add statistical context
+    if not df.empty:
+        numeric_cols = df.select_dtypes(include=[np.number]).columns
+        if len(numeric_cols) > 0:
+            enhanced['statistical_summary'] = {
+                'numeric_columns': len(numeric_cols),
+                'total_records': len(df),
+                'missing_data_percentage': (df.isnull().sum().sum() / (len(df) * len(df.columns))) * 100,
+                'key_metrics': {col: {'mean': df[col].mean(), 'std': df[col].std()} 
+                               for col in numeric_cols[:3]}  # Top 3 numeric columns
+            }
     
-    # Detect date/time columns
-    date_columns = []
-    for col in df.columns:
-        if df[col].dtype == 'datetime64[ns]' or 'date' in col.lower() or 'time' in col.lower():
-            try:
-                pd.to_datetime(df[col])
-                date_columns.append(col)
-            except:
-                continue
-    
-    if date_columns:
-        temporal_insights["has_temporal_data"] = True
-        temporal_insights["date_columns"] = date_columns
-        
-        # Analyze temporal patterns for the first date column
-        primary_date_col = date_columns[0]
-        df_temp = df.copy()
-        df_temp[primary_date_col] = pd.to_datetime(df_temp[primary_date_col])
-        
-        temporal_insights["temporal_analysis"] = {
-            "date_range": {
-                "start": df_temp[primary_date_col].min().strftime('%Y-%m-%d'),
-                "end": df_temp[primary_date_col].max().strftime('%Y-%m-%d')
-            },
-            "time_span_days": int((df_temp[primary_date_col].max() - df_temp[primary_date_col].min()).days),
-            "frequency": detect_temporal_frequency(df_temp[primary_date_col])
+    # Add categorical context
+    categorical_cols = df.select_dtypes(include=['object', 'category']).columns
+    if len(categorical_cols) > 0:
+        enhanced['categorical_summary'] = {
+            'categorical_columns': len(categorical_cols),
+            'unique_values': {col: df[col].nunique() for col in categorical_cols[:3]}
         }
     
-    return temporal_insights
-
+    # Merge with intelligence analysis
+    enhanced['ai_intelligence'] = intelligence
+    
+    return enhanced
 
-def assess_data_quality(df: pd.DataFrame) -> Dict[str, Any]:
+def generate_report_draft(buf, name: str, ctx: str, uid: str, project_id: str, bucket):
     """
-    Assesses data quality and identifies potential issues.
+    Enhanced autonomous report generation with intelligent narrative creation
     """
-    quality_metrics = {
-        "overall_quality_score": 0,
-        "quality_issues": [],
-        "data_completeness": 0,
-        "data_consistency": {}
-    }
+    logging.info(f"Generating autonomous report draft for project {project_id}")
     
-    # Completeness assessment with JSON-safe conversion
-    completeness = float((1 - df.isnull().sum().sum() / (len(df) * len(df.columns))) * 100)
-    quality_metrics["data_completeness"] = completeness
+    df = load_dataframe_safely(buf, name)
+    llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", google_api_key=API_KEY, temperature=0.1)
     
-    # Identify quality issues
-    if completeness < 95:
-        quality_metrics["quality_issues"].append("Missing data detected")
+    # Build enhanced context with AI intelligence
+    ctx_dict = {"shape": df.shape, "columns": list(df.columns), "user_ctx": ctx}
+    enhanced_ctx = enhance_data_context(df, ctx_dict)
     
-    # Check for duplicates with JSON-safe conversion
-    duplicate_rows = int(df.duplicated().sum())
-    if duplicate_rows > 0:
-        quality_metrics["quality_issues"].append(f"{duplicate_rows} duplicate rows found")
+    # Get AI intelligence analysis
+    intelligence = analyze_data_intelligence(df, ctx_dict)
     
-    # Check for inconsistent data types
-    for col in df.columns:
-        if df[col].dtype == 'object':
-            if df[col].str.isnumeric().any() and not df[col].str.isnumeric().all():
-                quality_metrics["quality_issues"].append(f"Inconsistent data types in {col}")
+    # Generate autonomous prompt
+    report_prompt = create_autonomous_prompt(df, enhanced_ctx, intelligence)
     
-    # Calculate overall quality score with JSON-safe conversion
-    base_score = 100.0
-    base_score -= (100 - completeness) * 0.5  # Penalize missing data
-    base_score -= len(quality_metrics["quality_issues"]) * 5  # Penalize each quality issue
-    quality_metrics["overall_quality_score"] = float(max(0, base_score))
+    # Generate the report
+    md = llm.invoke(report_prompt).content
     
-    return quality_metrics
-
-
-def infer_business_context(df: pd.DataFrame, domain_analysis: Dict[str, Any]) -> Dict[str, Any]:
-    """
-    Infers business context and potential use cases based on the data characteristics.
-    """
-    domain = domain_analysis["primary_domain"]
+    # Extract and process charts
+    chart_descs = extract_chart_tags(md)[:MAX_CHARTS]
+    chart_urls = {}
+    chart_generator = ChartGenerator(llm, df)
     
-    context_mapping = {
-        "financial": {
-            "key_metrics": ["Revenue", "Profit", "Cost", "ROI"],
-            "typical_analyses": ["Trend analysis", "Profitability analysis", "Budget vs actual"],
-            "stakeholders": ["CFO", "Finance team", "Executive leadership"]
-        },
-        "survey": {
-            "key_metrics": ["Satisfaction scores", "Response rates", "Sentiment"],
-            "typical_analyses": ["Satisfaction analysis", "Demographic breakdown", "Correlation analysis"],
-            "stakeholders": ["Marketing team", "Product managers", "Customer success"]
-        },
-        "scientific": {
-            "key_metrics": ["Statistical significance", "Effect size", "Confidence intervals"],
-            "typical_analyses": ["Hypothesis testing", "Regression analysis", "Experimental validation"],
-            "stakeholders": ["Researchers", "Scientists", "Academic community"]
-        },
-        "marketing": {
-            "key_metrics": ["Conversion rates", "Customer acquisition cost", "Campaign ROI"],
-            "typical_analyses": ["Campaign performance", "Customer segmentation", "Attribution analysis"],
-            "stakeholders": ["Marketing team", "CMO", "Sales team"]
-        }
-    }
+    for desc in chart_descs:
+        # Create a safe key for Firebase
+        safe_desc = sanitize_for_firebase_key(desc)
+        
+        # Replace the original description in the markdown with the safe one
+        md = md.replace(f'<generate_chart: "{desc}">', f'<generate_chart: "{safe_desc}">')
+        md = md.replace(f'<generate_chart: {desc}>', f'<generate_chart: "{safe_desc}">') # Handle no quotes case
+        
+        with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as temp_file:
+            img_path = Path(temp_file.name)
+            try:
+                chart_spec = chart_generator.generate_chart_spec(desc) # Still generate spec from original desc
+                if execute_chart_spec(chart_spec, df, img_path):
+                    blob_name = f"sozo_projects/{uid}/{project_id}/charts/{uuid.uuid4().hex}.png"
+                    blob = bucket.blob(blob_name)
+                    blob.upload_from_filename(str(img_path))
+                    
+                    # Use the safe key in the dictionary
+                    chart_urls[safe_desc] = blob.public_url
+                    logging.info(f"Uploaded chart '{desc}' to {blob.public_url} with safe key '{safe_desc}'")
+            finally:
+                if os.path.exists(img_path):
+                    os.unlink(img_path)
     
-    return context_mapping.get(domain, {
-        "key_metrics": ["Performance indicators", "Trends", "Patterns"],
-        "typical_analyses": ["Descriptive analysis", "Trend identification", "Pattern recognition"],
-        "stakeholders": ["Business stakeholders", "Decision makers"]
-    })
-
-
-def generate_intelligent_report(llm, autonomous_context: Dict[str, Any]) -> str:
-    """
-    Generates an intelligent, domain-appropriate report with organic storytelling.
-    """
-    # Create truly autonomous prompt that lets AI decide everything
-    enhanced_prompt = f"""
-You are a world-class data analyst who has just been handed this dataset to analyze. Look at the data characteristics and tell me the most compelling story you can find.
-
-**DATASET CONTEXT:**
-{json.dumps(autonomous_context, indent=2)}
-
-**YOUR MISSION:**
-Analyze this data like you would if a CEO walked into your office and said "I need to understand what this data is telling us." Write a report that would make them say "This is exactly what I needed to know."
+    return {"raw_md": md, "chartUrls": chart_urls}
 
-**GUIDELINES:**
-- Don't follow a rigid structure - let the data guide your narrative
-- Choose your own headings and sections based on what the data reveals
-- Write like you're presenting findings to someone who needs to make important decisions
-- Include specific numbers and insights that matter
-- Insert chart recommendations like: `<generate_chart: "chart_type | description">`
-- Valid chart types: bar, pie, line, scatter, hist, box, heatmap
-- Only recommend charts that truly support your narrative
+# Additional helper functions for the autonomous system
 
-**FORGET TEMPLATES - TELL THE STORY:**
-What's the most interesting, important, or surprising thing this data reveals? Start there and build your entire report around that central insight. Make it compelling, make it actionable, make it memorable.
+def detect_data_relationships(df: pd.DataFrame) -> Dict[str, Any]:
+    """Detect relationships and patterns in the data"""
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+    relationships = {}
+    
+    if len(numeric_cols) > 1:
+        corr_matrix = df[numeric_cols].corr()
+        # Find strong correlations (> 0.7 or < -0.7)
+        strong_correlations = []
+        for i in range(len(corr_matrix.columns)):
+            for j in range(i+1, len(corr_matrix.columns)):
+                corr_val = corr_matrix.iloc[i, j]
+                if abs(corr_val) > 0.7:
+                    strong_correlations.append({
+                        'var1': corr_matrix.columns[i],
+                        'var2': corr_matrix.columns[j],
+                        'correlation': corr_val
+                    })
+        relationships['strong_correlations'] = strong_correlations
+    
+    return relationships
 
-Be the data analyst who gets promoted because they don't just present data - they reveal insights that drive business decisions.
-"""
+def identify_key_metrics(df: pd.DataFrame, domain: str) -> List[str]:
+    """Identify the most important metrics based on domain and data characteristics"""
+    numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    
+    domain_priorities = {
+        'financial': ['revenue', 'profit', 'cost', 'amount', 'price', 'margin'],
+        'survey': ['rating', 'score', 'satisfaction', 'response'],
+        'marketing': ['conversion', 'click', 'impression', 'engagement'],
+        'operational': ['efficiency', 'utilization', 'throughput', 'performance']
+    }
+    
+    priorities = domain_priorities.get(domain, [])
+    key_metrics = []
+    
+    # Match column names with domain priorities
+    for col in numeric_cols:
+        col_lower = col.lower()
+        for priority in priorities:
+            if priority in col_lower:
+                key_metrics.append(col)
+                break
+    
+    # If no matches, use columns with highest variance (most interesting)
+    if not key_metrics and numeric_cols:
+        variances = df[numeric_cols].var().sort_values(ascending=False)
+        key_metrics = variances.head(3).index.tolist()
+    
+    return key_metrics[:5]  # Return top 5 key metrics
 
 
 # Removed - no longer needed since we're letting AI decide everything organically