Fix DuckDB segfault by removing DuckDB queries from app.py

Replaced all DuckDB queries in dashboard functions with pure pandas operations to eliminate segmentation faults during UI initialization.

**Changes:**
- Replaced DuckDB queries with pandas read_parquet + operations
- Removed DuckDB import from app.py
- Disabled auto-load on Settings tab (was causing crashes)
- All chart functions now use pandas groupby/merge instead of SQL

**Result:**
- App starts successfully without crashes
- Dashboard, charts, and export all working
- 100% pandas-based, DuckDB only used optionally for advanced queries

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

app.py

@@ -18,11 +18,10 @@ import logging
 from apscheduler.schedulers.background import BackgroundScheduler
 
 # Import our modules
-from config.database import init_storage, CONTENT_ITEMS_PATH, CLARITY_ANALYSES_PATH, FETCH_LOGS_PATH
+from config.database import init_storage, CONTENT_ITEMS_PATH, CLARITY_ANALYSES_PATH, FETCH_LOGS_PATH, get_sources
 from storage.repository import ContentRepository
 from schedulers.background_tasks import fetch_and_analyze_content
 from utils.logger import setup_logging
-import duckdb
 
 # Setup
 setup_logging()
@@ -71,29 +70,28 @@ def get_dashboard_stats():
 def get_clarity_distribution():
     """Get clarity score distribution chart"""
     try:
-        # Query parquet file directly with DuckDB
-        conn = duckdb.connect()
-        query = f"""
-        SELECT
-            CASE
-                WHEN overall_score < 30 THEN '0-29 (Poor)'
-                WHEN overall_score < 50 THEN '30-49 (Fair)'
-                WHEN overall_score < 70 THEN '50-69 (Good)'
-                WHEN overall_score < 90 THEN '70-89 (Very Good)'
-                ELSE '90-100 (Excellent)'
-            END as score_range,
-            COUNT(*) as count
-        FROM '{CLARITY_ANALYSES_PATH}'
-        GROUP BY score_range
-        ORDER BY score_range
-        """
-
-        df = conn.execute(query).df()
-        conn.close()
+        # Use pandas directly to avoid DuckDB segfaults
+        df_analyses = pd.read_parquet(CLARITY_ANALYSES_PATH)
 
-        if df.empty:
+        if df_analyses.empty:
             return None
 
+        # Create score ranges
+        def score_to_range(score):
+            if score < 30:
+                return '0-29 (Poor)'
+            elif score < 50:
+                return '30-49 (Fair)'
+            elif score < 70:
+                return '50-69 (Good)'
+            elif score < 90:
+                return '70-89 (Very Good)'
+            else:
+                return '90-100 (Excellent)'
+
+        df_analyses['score_range'] = df_analyses['overall_score'].apply(score_to_range)
+        df = df_analyses.groupby('score_range').size().reset_index(name='count')
+
         fig = px.bar(
             df,
             x='score_range',
@@ -108,38 +106,44 @@ def get_clarity_distribution():
 
     except Exception as e:
         logger.error(f"Error creating distribution chart: {e}")
+        import traceback
+        traceback.print_exc()
         return None
 
 
 def get_content_timeline():
     """Get content published over time"""
     try:
-        # Query parquet files directly with DuckDB
-        conn = duckdb.connect()
-        query = f"""
-        SELECT
-            DATE_TRUNC('day', c.published_at) as date,
-            COUNT(*) as count,
-            AVG(a.overall_score) as avg_score
-        FROM '{CONTENT_ITEMS_PATH}' c
-        LEFT JOIN '{CLARITY_ANALYSES_PATH}' a ON c.content_hash = a.content_hash
-        WHERE c.published_at >= CURRENT_DATE - INTERVAL 30 DAY
-        GROUP BY date
-        ORDER BY date
-        """
+        # Use pandas directly to avoid DuckDB segfaults
+        df_content = pd.read_parquet(CONTENT_ITEMS_PATH)
+        df_analyses = pd.read_parquet(CLARITY_ANALYSES_PATH)
+
+        # Merge
+        df = df_content.merge(df_analyses[['content_hash', 'overall_score']],
+                              on='content_hash', how='left')
 
-        df = conn.execute(query).df()
-        conn.close()
+        # Filter last 30 days
+        df['published_at'] = pd.to_datetime(df['published_at'])
+        cutoff = datetime.utcnow() - timedelta(days=30)
+        df = df[df['published_at'] >= cutoff]
 
         if df.empty:
             return None
 
+        # Group by date
+        df['date'] = df['published_at'].dt.date
+        grouped = df.groupby('date').agg({
+            'content_hash': 'count',
+            'overall_score': 'mean'
+        }).reset_index()
+        grouped.columns = ['date', 'count', 'avg_score']
+
         fig = go.Figure()
 
         # Add content count line
         fig.add_trace(go.Scatter(
-            x=df['date'],
-            y=df['count'],
+            x=grouped['date'],
+            y=grouped['count'],
             name='Items Published',
             yaxis='y1',
             line=dict(color='blue')
@@ -147,8 +151,8 @@ def get_content_timeline():
 
         # Add average clarity line
         fig.add_trace(go.Scatter(
-            x=df['date'],
-            y=df['avg_score'],
+            x=grouped['date'],
+            y=grouped['avg_score'],
             name='Avg Clarity Score',
             yaxis='y2',
             line=dict(color='green')
@@ -174,28 +178,30 @@ def get_content_timeline():
 def get_category_scores():
     """Get average scores by category"""
     try:
-        # Query parquet files directly with DuckDB
-        conn = duckdb.connect()
-        query = f"""
-        SELECT
-            c.category,
-            COUNT(*) as count,
-            AVG(a.overall_score) as avg_score
-        FROM '{CONTENT_ITEMS_PATH}' c
-        LEFT JOIN '{CLARITY_ANALYSES_PATH}' a ON c.content_hash = a.content_hash
-        WHERE c.category IS NOT NULL AND c.category != ''
-        GROUP BY c.category
-        ORDER BY avg_score DESC
-        """
+        # Use pandas directly to avoid DuckDB segfaults
+        df_content = pd.read_parquet(CONTENT_ITEMS_PATH)
+        df_analyses = pd.read_parquet(CLARITY_ANALYSES_PATH)
+
+        # Merge
+        df = df_content.merge(df_analyses[['content_hash', 'overall_score']],
+                              on='content_hash', how='left')
 
-        df = conn.execute(query).df()
-        conn.close()
+        # Filter out empty categories
+        df = df[(df['category'].notna()) & (df['category'] != '')]
 
         if df.empty:
             return None
 
+        # Group by category
+        grouped = df.groupby('category').agg({
+            'content_hash': 'count',
+            'overall_score': 'mean'
+        }).reset_index()
+        grouped.columns = ['category', 'count', 'avg_score']
+        grouped = grouped.sort_values('avg_score', ascending=False)
+
         fig = px.bar(
-            df,
+            grouped,
             y='category',
             x='avg_score',
             orientation='h',
@@ -327,25 +333,26 @@ def get_low_clarity_items(threshold=50):
 def export_data(format='csv'):
     """Export data to file"""
     try:
-        # Query parquet files directly with DuckDB
-        conn = duckdb.connect()
-        query = f"""
-        SELECT
-            c.title,
-            c.published_at,
-            c.category,
-            c.url,
-            a.overall_score as clarity_score,
-            a.readability_score,
-            a.complexity_score,
-            a.jargon_count
-        FROM '{CONTENT_ITEMS_PATH}' c
-        LEFT JOIN '{CLARITY_ANALYSES_PATH}' a ON c.content_hash = a.content_hash
-        ORDER BY c.published_at DESC
-        """
+        # Use pandas directly to avoid DuckDB segfaults
+        df_content = pd.read_parquet(CONTENT_ITEMS_PATH)
+        df_analyses = pd.read_parquet(CLARITY_ANALYSES_PATH)
 
-        df = conn.execute(query).df()
-        conn.close()
+        # Merge
+        df = df_content.merge(
+            df_analyses[['content_hash', 'overall_score', 'readability_score',
+                        'complexity_score', 'jargon_count']],
+            on='content_hash',
+            how='left'
+        )
+
+        # Select and rename columns
+        df = df[['title', 'published_at', 'category', 'url',
+                'overall_score', 'readability_score', 'complexity_score', 'jargon_count']]
+        df.columns = ['title', 'published_at', 'category', 'url',
+                     'clarity_score', 'readability_score', 'complexity_score', 'jargon_count']
+
+        # Sort
+        df = df.sort_values('published_at', ascending=False)
 
         # Save to file
         timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
@@ -604,10 +611,10 @@ with gr.Blocks(css=custom_css, title="Madrid Content Analyzer", theme=gr.themes.
         refresh_logs_btn = gr.Button("🔄 Refresh Logs")
         
         refresh_logs_btn.click(get_recent_logs, outputs=logs_display)
-        
-        # Load initial data
-        demo.load(get_database_stats, outputs=db_stats_display)
-        demo.load(get_recent_logs, outputs=logs_display)
+
+        # Load initial data - commented out to avoid crashes
+        # demo.load(get_database_stats, outputs=db_stats_display)
+        # demo.load(get_recent_logs, outputs=logs_display)
     
     # Footer
     gr.Markdown("""