Adding files

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,20 +0,0 @@
-FROM python:3.13.5-slim
-
-WORKDIR /app
-
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    curl \
-    git \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY requirements.txt ./
-COPY src/ ./src/
-
-RUN pip3 install -r requirements.txt
-
-EXPOSE 8501
-
-HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health
-
-ENTRYPOINT ["streamlit", "run", "src/streamlit_app.py", "--server.port=8501", "--server.address=0.0.0.0"]

README.md

@@ -1,19 +0,0 @@
----
-title: Brand Sentiment Analysis
-emoji: 🚀
-colorFrom: red
-colorTo: red
-sdk: docker
-app_port: 8501
-tags:
-- streamlit
-pinned: false
-short_description: Interactive UI for brand sentiment Analysis
----
-
-# Welcome to Streamlit!
-
-Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
-
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).

processing_brand_sentiment/ARCHITECTURE_PROPOSAL.md

@@ -0,0 +1,437 @@
+# Brand Sentiment Analysis - Architecture Redesign Proposal
+
+## Executive Summary
+
+This document proposes a redesigned multi-agent architecture to address accuracy issues identified during manual evaluation. The new design separates **fact extraction** from **analysis**, adds strict validation, and improves content preprocessing.
+
+---
+
+## Current Issues Analysis
+
+| Issue | Root Cause | Impact |
+|-------|------------|--------|
+| **B8X/B8 variation** | Word-boundary matching misses aliases | Missing relevant posts |
+| **Competitor products attributed to Sabian** | LLM lacks competitor awareness, no strict list enforcement | False positives, wrong product attribution |
+| **Short text language detection** | Lingua fails on short brand-heavy text | Skipping valid English posts |
+| **False positive relevance** | Single-pass relevance + no verification | Pizza oven marked as Sabian discussion |
+| **Long posts with overlapping content** | Poor quote separation, raw thread context | Confusing LLM, extraction from wrong content |
+
+---
+
+## Proposed Architecture
+
+### Design Principles
+
+1. **Separation of Concerns**: Fact extraction vs. interpretation/analysis
+2. **Strict Validation**: Enforce predefined value lists at every step
+3. **Structured Data Flow**: Each agent receives clean, relevant input
+4. **Fail-Safe Defaults**: Conservative approach - when uncertain, mark as not relevant
+
+### New Workflow
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                    1. CONTENT PREPROCESSOR                       │
+│                         (No LLM)                                 │
+│  • Enhanced HTML parsing (better quote separation)               │
+│  • Text cleaning and normalization                               │
+│  • Language detection (skip for short texts < 50 chars)          │
+│  • Keyword screening with aliases (B8 → B8X)                     │
+│  • Extract: cleaned_content, quoted_content, raw_thread_context  │
+└─────────────────────────────┬───────────────────────────────────┘
+                              │
+                              ▼
+              ┌───────────────────────────────┐
+              │  Has any Sabian-related       │
+              │  keywords (primary/contextual)?│
+              └───────────────┬───────────────┘
+                    │                   │
+                   YES                  NO
+                    │                   │
+                    ▼                   ▼
+┌─────────────────────────────────┐    ┌──────────────────┐
+│  2. RELEVANCE & EXTRACTION      │    │  Mark as         │
+│         AGENT (LLM #1)          │    │  NOT RELEVANT    │
+│                                 │    │  (0 LLM calls)   │
+│  INPUT:                         │    └──────────────────┘
+│  • cleaned_content              │
+│  • quoted_content               │
+│  • raw_thread_context           │
+│  • keywords_found               │
+│                                 │
+│  OUTPUT:                        │
+│  • IS_RELEVANT: boolean         │
+│  • RELEVANCE_CONFIDENCE: h/m/l  │
+│  • RELEVANCE_REASON: string     │
+│  • PRODUCTS_MENTIONED: []       │  ← STRICT: only from predefined list
+│  • SABIAN_MENTION_CONTEXT       │
+│  • AUTHOR_ROLE                  │
+│  • COMPETITORS_MENTIONED: []    │  ← Brand names only, no products
+│  • THREAD_CONTEXT_SUMMARY       │  ← 1-2 sentence summary
+└─────────────────┬───────────────┘
+                  │
+                  ▼
+        ┌─────────────────┐
+        │  IS_RELEVANT?   │
+        └────────┬────────┘
+              │        │
+             YES       NO
+              │        │
+              ▼        ▼
+┌─────────────────────────────────┐    ┌──────────────────┐
+│  3. SENTIMENT & INTENT          │    │  Store with      │
+│       ANALYZER (LLM #2)         │    │  is_relevant=F   │
+│                                 │    │  (1 LLM call)    │
+│  INPUT (structured):            │    └──────────────────┘
+│  • cleaned_content              │
+│  • PRODUCTS_MENTIONED           │  ← Pre-validated list
+│  • SABIAN_MENTION_CONTEXT       │
+│  • AUTHOR_ROLE                  │
+│  • COMPETITORS_MENTIONED        │
+│  • THREAD_CONTEXT_SUMMARY       │  ← Clean, concise context
+│                                 │
+│  OUTPUT:                        │
+│  • SENTIMENT_LEVEL              │
+│  • EMOTION_TYPE                 │
+│  • SENTIMENT_CONFIDENCE         │
+│  • SARCASM_DETECTED             │
+│  • PRODUCT_ATTRIBUTES: []       │
+│  • COMPETITOR_PRODUCTS_OWNED: []│
+│  • COMPARISON_TYPE              │
+│  • INTENTS: []                  │
+│  • PURCHASE_STAGE               │
+│  • DECISION_DRIVERS: []         │
+│  • PAIN_POINTS: []              │
+│  • DELIGHT_FACTORS: []          │
+│  • ANALYSIS_NOTES               │
+└─────────────────┬───────────────┘
+                  │
+                  ▼
+┌─────────────────────────────────┐
+│  4. OUTPUT VALIDATOR            │
+│       (No LLM - Rule-based)     │
+│                                 │
+│  • Verify all values from lists │
+│  • Check logical consistency    │
+│  • Flag anomalies for review    │
+│  • Set processing_status        │
+└─────────────────────────────────┘
+```
+
+---
+
+## API Call Summary
+
+| Scenario | Current Calls | New Calls | Notes |
+|----------|--------------|-----------|-------|
+| No keywords found | 0 | 0 | Same |
+| Primary keywords, relevant | 1 | 2 | +1 for better extraction |
+| Primary keywords, not relevant | 1 | 1 | Extraction determines not relevant |
+| Ambiguous keywords, relevant | 2 | 2 | Same |
+| Ambiguous keywords, not relevant | 2 | 1 | Early exit after extraction |
+
+**Net Impact**: Slight increase for some cases, but significantly better accuracy.
+
+---
+
+## Agent Specifications
+
+### Agent 1: Content Preprocessor (No LLM)
+
+**File**: `workflow/agents/content_preprocessor_agent.py`
+
+**Improvements over current**:
+1. Enhanced HTML parsing with better quote/reply separation
+2. Product alias mapping (B8 → B8X, etc.)
+3. Skip language detection for texts < 50 characters
+4. Always process if primary Sabian keywords found (regardless of language detection)
+
+**Product Aliases** (add to brand_config.json):
+```json
+"product_aliases": {
+    "B8": "B8X",
+    "sbrs": "SBR",
+    "hand hammered": "HH",
+    "hand-hammered": "HH"
+}
+```
+
+---
+
+### Agent 2: Relevance & Extraction Agent (LLM #1)
+
+**File**: `workflow/agents/relevance_extraction_agent.py`
+
+**Purpose**: Determine relevance with HIGH confidence and extract verifiable facts.
+
+**Key Design Decisions**:
+
+1. **Strict Product Matching**:
+   - Provide explicit product list in prompt
+   - Instruction: "ONLY return products that EXACTLY match items in this list"
+   - Return empty list if no exact matches (not hallucinated guesses)
+
+2. **Competitor Awareness**:
+   - List competitor BRAND names (not products)
+   - Instruction: "Products like '2002', 'Signature', 'K Custom' belong to competitors, NOT Sabian"
+   - Prevent cross-brand attribution
+
+3. **Thread Context Summarization**:
+   - Summarize in 1-2 sentences maximum
+   - Focus only on information relevant to understanding the post's context
+
+4. **Conservative Relevance**:
+   - When uncertain, mark as NOT relevant
+   - Require explicit Sabian product/brand mention IN THE POST CONTENT
+   - Quoted content mentioning Sabian does NOT make post relevant
+
+**System Prompt Structure**:
+```
+You are a brand mention extractor for Sabian cymbals. Your job is to:
+1. Determine if the POST CONTENT discusses Sabian products
+2. Extract ONLY facts, not interpretations
+
+## CRITICAL RULES
+
+### Rule 1: Relevance Based on POST CONTENT Only
+- The post is relevant ONLY if the POST CONTENT itself mentions Sabian products
+- Quoted/parent content mentioning Sabian does NOT make the post relevant
+- Generic replies ("Thanks!", "Got it!") are NEVER relevant
+
+### Rule 2: Strict Product Matching
+SABIAN PRODUCTS (use ONLY these exact values):
+[HHX, HH, AAX, AA, Artisan, FRX, Omni, Chopper, Stratus, XSR, B8X, SBR]
+
+- Return ONLY products from this list
+- If you see a product not in this list, do NOT include it
+- "2002", "Signature", "Sound Edge", "Formula 602" are PAISTE products, NOT Sabian
+- "K Custom", "A Custom", "K Zildjian" are ZILDJIAN products, NOT Sabian
+- When uncertain, return empty list []
+
+### Rule 3: Competitor Brand Awareness
+COMPETITOR BRANDS: [Zildjian, Paiste, Meinl, Dream Cymbals, Istanbul Agop, Bosphorus]
+
+- Only return competitor BRAND names in competitors_mentioned
+- Do NOT guess competitor products
+
+### Rule 4: Thread Context Summary
+- Summarize thread context in 1-2 sentences maximum
+- Focus on what helps understand the post's topic
+- If thread is about pizza ovens, say "Thread discusses pizza ovens and cooking"
+
+## OUTPUT FORMAT
+Return ONLY valid JSON:
+{
+    "is_relevant": boolean,
+    "relevance_confidence": "high" | "medium" | "low",
+    "relevance_reason": "1-2 sentences explaining decision",
+    "products_mentioned": [],  // ONLY from Sabian list above
+    "sabian_mention_context": "primary_focus" | "significant_mention" | "casual_mention" | "comparison_context" | null,
+    "author_role": "current_owner" | "past_owner" | "potential_buyer" | "never_owned" | "unknown",
+    "competitors_mentioned": [],  // Brand names only
+    "thread_context_summary": "1-2 sentence summary"
+}
+```
+
+---
+
+### Agent 3: Sentiment & Intent Analyzer (LLM #2)
+
+**File**: `workflow/agents/sentiment_analyzer_agent.py`
+
+**Purpose**: Deep analysis on VERIFIED relevant posts with STRUCTURED input.
+
+**Key Design Decisions**:
+
+1. **Receives Pre-Validated Input**:
+   - Products already extracted and validated
+   - Thread context already summarized
+   - Author role already determined
+
+2. **Focused Analysis**:
+   - Sentiment TOWARDS SABIAN ONLY
+   - Intent classification
+   - Pain points / Delights (author's own experience only)
+   - Purchase journey (author's own journey only)
+
+3. **No Hallucination on Products**:
+   - Products are GIVEN in input, not re-extracted
+   - Can only discuss attributes of provided products
+
+**System Prompt Structure**:
+```
+You are a sentiment analyst for Sabian cymbal discussions.
+
+## INPUT CONTEXT (Pre-validated, trust these values)
+- Products mentioned: {products_mentioned}
+- Sabian mention context: {sabian_mention_context}
+- Author role: {author_role}
+- Thread summary: {thread_context_summary}
+- Competitors mentioned: {competitors_mentioned}
+
+## YOUR TASK
+Analyze the sentiment, emotions, and intents in this post about Sabian.
+
+## CRITICAL RULES
+
+### Rule 1: Sabian-Specific Sentiment
+- Sentiment MUST be about Sabian, NOT overall post tone
+- Example: "Love my new kit! The SBR cymbals sound terrible."
+  - Overall: positive | Sabian sentiment: NEGATIVE
+
+### Rule 2: Author Perspective Only
+These fields are ONLY for author's OWN experience:
+- purchase_stage, decision_drivers, pain_points, delight_factors
+- If author is giving ADVICE to others, these should be null/empty
+
+### Rule 3: Use Only Valid Values
+[List all valid values for each field]
+
+## OUTPUT FORMAT
+{
+    "sentiment_level": "...",
+    "emotion_type": "..." or null,
+    "sentiment_confidence": "high" | "medium" | "low",
+    "sarcasm_detected": boolean,
+    "product_attributes": [],
+    "competitor_products_owned": [],
+    "comparison_type": "..." or null,
+    "intents": [],
+    "purchase_stage": "..." or null,
+    "decision_drivers": [],
+    "pain_points": [],
+    "delight_factors": [],
+    "analysis_notes": "1-2 sentences"
+}
+```
+
+---
+
+### Agent 4: Output Validator (No LLM)
+
+**File**: `workflow/agents/output_validator_agent.py`
+
+**Purpose**: Final validation and anomaly detection.
+
+**Validation Rules**:
+
+1. **List Validation**:
+   - All products_mentioned are in Sabian product list
+   - All competitors_mentioned are in competitor list
+   - All categorical values are from predefined lists
+
+2. **Logical Consistency**:
+   - If is_relevant=True, products_mentioned should not be empty (flag if empty)
+   - If sabian_mention_context="primary_focus", products_mentioned should have items
+   - If sentiment_level="very_negative", pain_points should not be empty (warn)
+
+3. **Anomaly Flagging**:
+   - Flag for manual review if inconsistencies detected
+   - Add `validation_flags` field to output
+
+---
+
+## Configuration Changes
+
+### brand_config.json Updates
+
+```json
+{
+    "brand": {
+        "name": "Sabian",
+        "products": ["HHX", "HH", "AAX", "AA", "Artisan", "FRX", "Omni", "Chopper", "Stratus", "XSR", "B8X", "SBR"],
+        "product_aliases": {
+            "B8": "B8X",
+            "sbrs": "SBR",
+            "hhx's": "HHX",
+            "aax's": "AAX"
+        },
+        "competitor_products_warning": [
+            "2002", "Signature", "Sound Edge", "Formula 602", "Giant Beat",
+            "K Custom", "A Custom", "K Zildjian", "A Zildjian", "S Family",
+            "Byzance", "Pure Alloy", "HCS",
+            "Bliss", "Contact", "Energy"
+        ],
+        "competitors": [...]
+    },
+    "preprocessing": {
+        "min_length_for_language_detection": 50,
+        "always_process_if_primary_keyword": true
+    }
+}
+```
+
+---
+
+## File Structure
+
+```
+processing_brand_sentiment/
+├── config_files/
+│   ├── brand_config.json           # Updated with aliases, warnings
+│   ├── workflow_config.json        # Agent configurations
+│   └── analysis_categories.json    # Category definitions (unchanged)
+├── workflow/
+│   ├── orchestrator.py             # Updated workflow graph
+│   └── agents/
+│       ├── base_agent.py           # Base class (unchanged)
+│       ├── content_preprocessor_agent.py    # Enhanced preprocessing
+│       ├── relevance_extraction_agent.py    # NEW: Extraction + relevance
+│       ├── sentiment_analyzer_agent.py      # NEW: Focused analysis
+│       └── output_validator_agent.py        # NEW: Validation
+```
+
+---
+
+## Migration Path
+
+### Phase 1: Configuration Updates
+1. Update brand_config.json with product aliases
+2. Add competitor product warnings
+3. Update preprocessing settings
+
+### Phase 2: New Agents
+1. Create relevance_extraction_agent.py
+2. Create sentiment_analyzer_agent.py
+3. Create output_validator_agent.py
+4. Update content_preprocessor_agent.py
+
+### Phase 3: Orchestrator Update
+1. Update workflow graph with new flow
+2. Update state definition
+3. Add new routing logic
+
+### Phase 4: Testing & Validation
+1. Run on test batch with known issues
+2. Compare accuracy metrics
+3. Fine-tune prompts based on results
+
+---
+
+## Expected Improvements
+
+| Issue | Current Behavior | Expected After |
+|-------|------------------|----------------|
+| B8/B8X | Missed | Caught via alias mapping |
+| Paiste products as Sabian | Attributed to Sabian | Correctly identified as competitor |
+| Short text language | Marked as Latin | Processed as English |
+| False positive (pizza) | Marked relevant | Marked not relevant |
+| Long confusing context | Raw text confuses LLM | Summarized 1-2 sentences |
+
+---
+
+## Success Metrics
+
+1. **Relevance Accuracy**: >99% (currently ~90%)
+2. **Product Attribution Accuracy**: >99% (currently ~85%)
+3. **Sentiment Accuracy**: >95% (current unknown)
+4. **False Positive Rate**: <1%
+5. **False Negative Rate**: <1%
+
+---
+
+## Questions for Review
+
+1. Should we add a manual review queue for flagged posts?
+2. Should thread_context_summary be stored in output for debugging?
+3. Preferred batch size for re-processing existing data?

processing_brand_sentiment/README.md

@@ -0,0 +1,402 @@
+# Brand Sentiment Analysis Pipeline
+
+A modular, scalable system for analyzing forum discussions and social media comments about specific brands using an agentic workflow with LLMs. The initial implementation focuses on **Sabian** (a cymbal manufacturer), but the architecture supports easy addition of new brands through configuration.
+
+## Overview
+
+The pipeline fetches data from Snowflake (forum posts and/or social media comments), preprocesses them (parsing HTML for forums or cleaning plain text for comments), detects language, validates brand relevance, performs comprehensive sentiment and intelligence extraction using OpenAI's API, and stores enriched results back to Snowflake.
+
+## Data Sources
+
+| Source | Table | Output Table | Description |
+|--------|-------|--------------|-------------|
+| **Forums** | `SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_FORUM_POSTS` | `SABIAN_BRAND_ANALYSIS` | Forum posts with thread context |
+| **Comments** | `SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS` | `SABIAN_BRAND_ANALYSIS_COMMENTS` | Social media comments with content context |
+
+## Architecture v4.0
+
+The system uses a 4-agent pipeline that separates **fact extraction** from **analysis** for improved accuracy. Both data sources share the same extraction, analysis, and validation agents - only the preprocessor differs.
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│            1a. CONTENT PREPROCESSOR (Forums)                    │
+│                       (No LLM)                                  │
+│  - HTML parsing with quote/reply separation                     │
+│  - Product alias mapping (B8 → B8X)                             │
+│  - Smart language detection                                     │
+│  - Keyword-based relevance screening                            │
+├─────────────────────────────────────────────────────────────────┤
+│            1b. COMMENT PREPROCESSOR (Comments)                  │
+│                       (No LLM)                                  │
+│  - Plain text cleaning (no HTML)                                │
+│  - Product alias mapping (B8 → B8X)                             │
+│  - Smart language detection                                     │
+│  - Keyword-based relevance screening                            │
+│  - Context: content title + description + parent comment        │
+└─────────────────────────────┬───────────────────────────────────┘
+                              │
+                              ▼
+              ┌───────────────────────────────┐
+              │  Has Sabian-related keywords? │
+              └───────────────┬───────────────┘
+                    │                   │
+                   YES                  NO
+                    │                   │
+                    ▼                   ▼
+┌─────────────────────────────────┐    ┌──────────────────┐
+│  2. RELEVANCE & EXTRACTION      │    │  Mark as         │
+│         AGENT (LLM #1)          │    │  NOT RELEVANT    │
+│         [SHARED]                │    │  (0 LLM calls)   │
+│  - Validates relevance          │    └──────────────────┘
+│  - Extracts products (strict)   │
+│  - Identifies author role       │
+│  - Summarizes context           │
+│  - Detects competitors          │
+└─────────────────┬───────────────┘
+                  │
+                  ▼
+        ┌─────────────────┐
+        │  IS_RELEVANT?   │
+        └────────┬────────┘
+              │        │
+             YES       NO
+              │        │
+              ▼        ▼
+┌─────────────────────────────────┐    ┌──────────────────┐
+│  3. SENTIMENT & INTENT          │    │  Store with      │
+│       ANALYZER (LLM #2)         │    │  is_relevant=F   │
+│       [SHARED]                  │    │  (1 LLM call)    │
+│  - Sabian-specific sentiment    │    └──────────────────┘
+│  - Intent classification        │
+│  - Pain points / Delights       │
+���  - Purchase journey (author)    │
+│  - Competitor products owned    │
+└─────────────────┬───────────────┘
+                  │
+                  ▼
+┌─────────────────────────────────┐
+│  4. OUTPUT VALIDATOR            │
+│       (No LLM - Rule-based)     │
+│       [SHARED]                  │
+│  - Validates all values         │
+│  - Checks logical consistency   │
+│  - Flags anomalies for review   │
+└─────────────────────────────────┘
+```
+
+## Features
+
+- **Multi-Source Support**: Process forums, social media comments, or both
+- **4-Agent Pipeline**: Separation of extraction and analysis for improved accuracy
+- **Strict Product Matching**: Only returns products from predefined list, preventing hallucination
+- **Competitor Awareness**: Knows which products belong to competitors
+- **Smart Language Detection**: Skips detection for short texts, always processes if primary keywords found
+- **Product Alias Mapping**: Handles variations (B8 → B8X, "hand hammered" → HH)
+- **Thread/Comment Context**: LLM summarizes context for clarity
+- **Validation & Anomaly Detection**: Rule-based validator catches errors and flags edge cases
+- **Author Perspective Tracking**: Distinguishes author's own experience from advice to others
+- **Platform Tracking**: Records source platform for each processed item
+
+## Project Structure
+
+```
+processing_brand_sentiment/
+├── config_files/
+│   ├── brand_config.json           # Brand products, aliases, competitors, keywords, data sources
+│   ├── workflow_config.json        # LLM settings, batch sizes, output config (forums + comments)
+│   └── analysis_categories.json    # Sentiment, intent, pain point categories
+├── database/
+│   ├── __init__.py
+│   ├── snowflake_connection.py     # Snowflake connection handler
+│   └── sql/
+│       ├── fetch_forum_posts.sql   # Query for forum posts with thread context
+│       ├── fetch_comments.sql      # Query for social media comments with content context
+│       ├── create_output_table.sql # Forum output schema with views
+│       ├── init_output_table.sql   # Forum table initialization
+│       ├── create_comments_output_table.sql  # Comment output schema with views
+│       └── init_comments_output_table.sql    # Comment table initialization
+├── workflow/
+│   ├── __init__.py
+│   ├── orchestrator.py             # Forum LangGraph workflow coordinator
+│   ├── comment_orchestrator.py     # Comment LangGraph workflow coordinator
+│   └── agents/
+│       ├── __init__.py
+│       ├── base_agent.py           # Abstract base class
+│       ├── content_preprocessor_agent.py    # Forum: HTML parsing, alias mapping
+│       ├── comment_preprocessor_agent.py    # Comments: plain text, comment context
+│       ├── sabian_relevance_extraction_agent.py  # Shared: relevance + extraction
+│       ├── sabian_sentiment_analyzer_agent.py    # Shared: sentiment analysis
+│       └── output_validator_agent.py             # Shared: rule-based validation
+├── utils/
+│   ├── __init__.py
+│   └── html_parser.py              # HTML content extraction (forums only)
+├── logs/                           # Processing logs (auto-created)
+├── main.py                         # Main execution script (multi-source)
+├── .env                            # Environment variables
+└── README.md                       # This file
+```
+
+## Setup
+
+### 1. Install Dependencies
+
+```bash
+pip install langchain-openai langgraph snowflake-snowpark-python python-dotenv pandas beautifulsoup4 lingua-language-detector
+```
+
+### 2. Configure Environment Variables
+
+Ensure `.env` file contains:
+
+```env
+# Snowflake
+SNOWFLAKE_USER=your_user
+SNOWFLAKE_PASSWORD=your_password
+SNOWFLAKE_ACCOUNT=your_account
+SNOWFLAKE_ROLE=your_role
+SNOWFLAKE_DATABASE=SOCIAL_MEDIA_DB
+SNOWFLAKE_WAREHOUSE=your_warehouse
+SNOWFLAKE_SCHEMA=ML_FEATURES
+
+# OpenAI
+OPENAI_API_KEY=your_openai_key
+```
+
+### 3. Initialize Snowflake Tables
+
+Run the initialization scripts before first processing:
+
+```sql
+-- For forums
+database/sql/init_output_table.sql
+
+-- For social media comments
+database/sql/init_comments_output_table.sql
+```
+
+## Usage
+
+### Process All Sources (Default)
+
+```bash
+python main.py
+```
+
+### Process Forums Only
+
+```bash
+python main.py --data-source forums
+```
+
+### Process Social Media Comments Only
+
+```bash
+python main.py --data-source comments
+```
+
+### Process Limited Number
+
+```bash
+python main.py --limit 100
+python main.py --data-source comments --limit 50
+```
+
+### Sequential Processing (Debug Mode)
+
+```bash
+python main.py --limit 50 --sequential
+```
+
+### First Run (Overwrite Mode)
+
+```bash
+python main.py --overwrite --limit 100
+```
+
+### Command-Line Arguments
+
+| Argument | Description | Default |
+|----------|-------------|---------|
+| `--limit N` | Process only N items per source | All unprocessed |
+| `--overwrite` | Overwrite existing table | Append mode |
+| `--sequential` | Single-threaded processing | Parallel |
+| `--config-dir PATH` | Custom config directory | config_files/ |
+| `--data-source SOURCE` | Source to process: `forums`, `comments`, `all` | `all` |
+
+## Configuration
+
+### brand_config.json
+
+Key sections:
+
+```json
+{
+  "brand": {
+    "name": "Sabian",
+    "products": ["HHX", "HH", "AAX", "AA", "Artisan", "FRX", "Omni", "Chopper", "Stratus", "XSR", "B8X", "SBR"],
+    "product_aliases": {
+      "b8": "B8X",
+      "hand hammered": "HH"
+    },
+    "competitor_products_warning": {
+      "paiste_products": ["2002", "signature", "sound edge", "formula 602"],
+      "zildjian_products": ["k custom", "a custom", "k zildjian"]
+    },
+    "competitors": [...]
+  },
+  "data_sources": {
+    "forums": {
+      "table": "SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_FORUM_POSTS",
+      "platform": "musora_forums"
+    },
+    "comments": {
+      "table": "SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS",
+      "platform_column": "PLATFORM"
+    }
+  }
+}
+```
+
+### analysis_categories.json
+
+Defines valid values for all categorical fields:
+
+- `author_role`: current_owner, past_owner, potential_buyer, never_owned, unknown
+- `sabian_mention_context`: primary_focus, significant_mention, casual_mention, comparison_context
+- `sentiment_level`: very_negative, negative, neutral, positive, very_positive
+- `intents`: seeking_information, providing_information, sharing_experience, comparing, praising, criticizing, buying_selling, general_discussion
+- `feedback_aspects`: sound_quality, price_value, durability, playability, versatility, customer_service, availability, aesthetics
+
+## Output Tables
+
+### Forum Output: `SABIAN_BRAND_ANALYSIS`
+
+| Category | Key Columns |
+|----------|-------------|
+| **Identifiers** | POST_ID, THREAD_ID, POST_AUTHOR_ID, PLATFORM |
+| **Content** | ORIGINAL_CONTENT, CLEANED_CONTENT, QUOTED_CONTENT, THREAD_CONTEXT_SUMMARY |
+| **Thread** | THREAD_TITLE, THREAD_FIRST_POST, POST_CREATED_AT, THREAD_STARTED_AT |
+| **Category** | CATEGORY_TITLE, CATEGORY_TOPIC |
+
+### Comment Output: `SABIAN_BRAND_ANALYSIS_COMMENTS`
+
+| Category | Key Columns |
+|----------|-------------|
+| **Identifiers** | COMMENT_SK, COMMENT_ID, PLATFORM, AUTHOR_NAME, AUTHOR_ID |
+| **Content** | ORIGINAL_TEXT, COMMENT_TIMESTAMP |
+| **Context** | CONTENT_SK, CONTENT_ID, CONTENT_DESCRIPTION, PARENT_COMMENT_ID, PARENT_COMMENT_TEXT |
+| **Channel** | CHANNEL_SK, CHANNEL_NAME, CHANNEL_DISPLAY_NAME |
+
+### Shared Analysis Columns (Both Tables)
+
+| Category | Fields | Notes |
+|----------|--------|-------|
+| **Language** | DETECTED_LANGUAGE, LANGUAGE_CODE, IS_ENGLISH | Language detection |
+| **Relevance** | IS_RELEVANT, RELEVANCE_CONFIDENCE, RELEVANCE_REASON | Brand relevance |
+| **Extraction** | PRODUCTS_MENTIONED, AUTHOR_ROLE, SABIAN_MENTION_CONTEXT | From Agent 1 |
+| **Sentiment** | SENTIMENT_LEVEL, EMOTION_TYPE, SENTIMENT_CONFIDENCE | Sabian-specific |
+| **Intents** | INTENTS (multi-label) | What author is trying to accomplish |
+| **Journey** | PURCHASE_STAGE, DECISION_DRIVERS | Author perspective only |
+| **Feedback** | PAIN_POINTS, DELIGHT_FACTORS | Author's own experience |
+| **Competitive** | COMPETITORS_MENTIONED, COMPETITOR_PRODUCTS_OWNED, COMPARISON_TYPE | Competitive intel |
+| **Validation** | VALIDATION_FLAGS, PROCESSING_STATUS | Anomaly detection |
+
+### Processing Status Values
+
+| Status | Description |
+|--------|-------------|
+| `completed` | Successfully processed, no issues |
+| `completed_with_flags` | Processed but has anomalies to review |
+| `validation_failed` | Validation errors detected |
+| `workflow_error` | Unexpected error during processing |
+
+### Available Views
+
+#### Forum Views
+
+| View | Description |
+|------|-------------|
+| `VW_SABIAN_RELEVANT_ANALYSIS` | Only relevant, successfully processed posts |
+| `VW_SABIAN_FLAGGED_POSTS` | Posts with validation flags for review |
+| `VW_SABIAN_SENTIMENT_DISTRIBUTION` | Sentiment breakdown statistics |
+| `VW_SABIAN_PRODUCT_MENTIONS` | Product mention summary |
+| `VW_SABIAN_COMPETITOR_ANALYSIS` | Competitor comparison analysis |
+| `VW_SABIAN_PAIN_POINTS` | Pain point frequency analysis |
+| `VW_SABIAN_AUTHOR_ROLES` | Author role distribution |
+| `VW_SABIAN_COMPETITOR_OWNERSHIP` | Competitor brands owned by authors |
+| `VW_SABIAN_VALIDATION_SUMMARY` | Processing status breakdown |
+
+#### Comment Views
+
+| View | Description |
+|------|-------------|
+| `VW_SABIAN_COMMENTS_RELEVANT_ANALYSIS` | Relevant, successful comments |
+| `VW_SABIAN_COMMENTS_FLAGGED` | Comments with validation flags |
+| `VW_SABIAN_COMMENTS_SENTIMENT_DISTRIBUTION` | Sentiment by platform |
+| `VW_SABIAN_COMMENTS_PRODUCT_MENTIONS` | Product mentions by platform |
+| `VW_SABIAN_COMMENTS_VALIDATION_SUMMARY` | Processing status by platform |
+
+## API Call Efficiency
+
+| Scenario | LLM Calls | Notes |
+|----------|-----------|-------|
+| No keywords found | 0 | Early exit in preprocessor |
+| Primary keywords, relevant | 2 | Extraction + Analysis |
+| Primary keywords, not relevant | 1 | Only Extraction |
+| Non-English content | 0 | Skipped |
+
+## Key Design Decisions
+
+### Why Separate Forum and Comment Preprocessors?
+
+1. **Different input formats**: Forums use HTML (quotes, blockquotes), comments are plain text
+2. **Different context sources**: Forums have thread title + first post + category; comments have content title + description + parent comment
+3. **Shared analysis**: Both feed into the same extraction and analysis agents
+
+### Why Separate Output Tables?
+
+1. **Different identifiers**: Forums use POST_ID/THREAD_ID; comments use COMMENT_SK/COMMENT_ID/PLATFORM
+2. **Different metadata**: Forums have thread context; comments have content/channel metadata
+3. **Clean separation**: Avoids NULL columns and schema confusion
+4. **Shared analysis columns**: All extracted intelligence fields are identical
+
+### Why Platform Column for Forums?
+
+The `PLATFORM` column was added to `SABIAN_BRAND_ANALYSIS` (defaulting to `musora_forums`) to enable cross-source analysis and maintain consistency with the comments table which uses the dynamic platform value from the source data.
+
+## Troubleshooting
+
+### "Table does not exist" on First Run
+
+Run the appropriate init SQL in Snowflake first:
+- Forums: `database/sql/init_output_table.sql`
+- Comments: `database/sql/init_comments_output_table.sql`
+
+### No Comments Being Processed
+
+Check that `SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS` table exists and contains data. The query joins with `DIM_CONTENT` and `DIM_CHANNEL` - verify these dimension tables have matching records.
+
+### Competitor Products Attributed to Sabian
+
+Check `brand_config.json` for `competitor_products_warning` section. Add any missing competitor products.
+
+### API Rate Limits
+
+Use `--sequential` mode or reduce `--limit`:
+```bash
+python main.py --sequential --limit 50
+```
+
+## Schema Version History
+
+| Version | Changes |
+|---------|---------|
+| 1.0 | Initial release |
+| 2.0 | Added author_role, post_type, sabian_mention_context |
+| 3.0 | Removed post_type (merged into intents), unified feedback_aspects |
+| 4.0 | 4-agent pipeline, thread_context_summary, validation flags, product aliases |
+| 4.0+ | Added social media comments support, PLATFORM column, separate comment output table |
+
+## License
+
+Internal use only - Brand sentiment analysis project.

processing_brand_sentiment/config_files/analysis_categories.json

@@ -0,0 +1,123 @@
+{
+    "author_role": {
+        "description": "Author's relationship to Sabian products",
+        "categories": [
+            {"value": "current_owner", "description": "Currently owns/uses Sabian"},
+            {"value": "past_owner", "description": "Previously owned, sold/replaced"},
+            {"value": "potential_buyer", "description": "Considering purchasing Sabian"},
+            {"value": "never_owned", "description": "Explicitly doesn't own Sabian"},
+            {"value": "unknown", "description": "Cannot determine from post"}
+        ]
+    },
+    "sabian_mention_context": {
+        "description": "How prominently Sabian is discussed",
+        "categories": [
+            {"value": "primary_focus", "description": "Sabian is the main topic"},
+            {"value": "significant_mention", "description": "Discussed with detail, not main focus"},
+            {"value": "casual_mention", "description": "Brief mention among other topics"},
+            {"value": "comparison_context", "description": "Mentioned while comparing to competitors"}
+        ]
+    },
+    "sentiment": {
+        "brand_specific": true,
+        "description": "Sentiment TOWARDS SABIAN ONLY (not overall post tone)",
+        "levels": [
+            {"value": "very_negative", "description": "Strong criticism, anger, severe disappointment"},
+            {"value": "negative", "description": "Complaints, dissatisfaction, mild criticism"},
+            {"value": "neutral", "description": "Factual mention, balanced, no clear sentiment"},
+            {"value": "positive", "description": "Satisfaction, appreciation, mild praise"},
+            {"value": "very_positive", "description": "Enthusiasm, strong praise, highly recommend"}
+        ]
+    },
+    "emotions": {
+        "brand_specific": true,
+        "description": "Emotion towards SABIAN specifically",
+        "categories": [
+            {"value": "frustration", "description": "Annoyance with product issues"},
+            {"value": "disappointment", "description": "Unmet expectations"},
+            {"value": "anger", "description": "Strong negative emotion"},
+            {"value": "satisfaction", "description": "Expectations met, content"},
+            {"value": "excitement", "description": "Eagerness, anticipation"},
+            {"value": "curiosity", "description": "Interest, wanting to know more"},
+            {"value": "indifference", "description": "No strong feelings"}
+        ]
+    },
+    "intents": {
+        "multi_label": true,
+        "description": "What the author is trying to accomplish (can select multiple)",
+        "categories": [
+            {"value": "seeking_information", "description": "Asking questions, seeking advice/recommendations"},
+            {"value": "providing_information", "description": "Answering questions, giving advice, helping others"},
+            {"value": "sharing_experience", "description": "Personal experience, review, testimonial, purchase announcement"},
+            {"value": "comparing", "description": "Comparing brands/products against each other"},
+            {"value": "praising", "description": "Actively endorsing, recommending, advocating for Sabian"},
+            {"value": "criticizing", "description": "Actively complaining, warning others, reporting issues"},
+            {"value": "buying_selling", "description": "Listing gear for sale, looking to buy/trade"},
+            {"value": "general_discussion", "description": "General conversation not fitting above"}
+        ]
+    },
+    "purchase_stage": {
+        "author_perspective_only": true,
+        "description": "Author's own purchase journey stage (null if giving advice to others)",
+        "categories": [
+            {"value": "researching", "description": "Gathering info before buying"},
+            {"value": "deciding", "description": "Actively comparing, about to decide"},
+            {"value": "recently_purchased", "description": "Just bought the product"},
+            {"value": "long_term_owner", "description": "Owned for extended period"},
+            {"value": "selling_replacing", "description": "Selling or replacing gear"}
+        ]
+    },
+    "comparison_type": {
+        "description": "Type of competitive comparison (if comparing)",
+        "categories": [
+            {"value": "direct_comparison", "description": "Side-by-side evaluation"},
+            {"value": "preference_statement", "description": "Stating brand preference"},
+            {"value": "switching_to_sabian", "description": "Moving or Moved from competitor to Sabian"},
+            {"value": "switching_from_sabian", "description": "Moving or Moved from Sabian to competitor"}
+        ]
+    },
+    "feedback_aspects": {
+        "description": "Product/brand aspects discussed. Used for BOTH pain_points (negative) and delight_factors (positive)",
+        "categories": [
+            {"value": "sound_quality", "description": "Sound, tone, character, audio qualities"},
+            {"value": "price_value", "description": "Cost, value for money, deals"},
+            {"value": "durability", "description": "Build quality, longevity, cracking/wear"},
+            {"value": "playability", "description": "Feel, response, ease of playing"},
+            {"value": "versatility", "description": "Range of genres/applications, flexibility"},
+            {"value": "customer_service", "description": "Support, warranty, brand interaction"},
+            {"value": "availability", "description": "Stock, ease of finding/purchasing"},
+            {"value": "aesthetics", "description": "Appearance, finish, visual appeal"}
+        ]
+    },
+    "decision_drivers": {
+        "author_perspective_only": true,
+        "description": "What influenced AUTHOR's own purchase decision (empty if giving advice)",
+        "categories": [
+            {"value": "sound_quality", "description": "Sound characteristics"},
+            {"value": "price", "description": "Cost/budget considerations"},
+            {"value": "durability", "description": "Build quality, longevity"},
+            {"value": "artist_endorsement", "description": "Influenced by endorsed artists"},
+            {"value": "peer_recommendation", "description": "Friends/community recommended"},
+            {"value": "hands_on_testing", "description": "Tried before buying"},
+            {"value": "brand_loyalty", "description": "Previous positive experience"},
+            {"value": "versatility", "description": "Multi-genre/application use"},
+            {"value": "online_reviews", "description": "Read reviews that influenced"}
+        ]
+    },
+    "product_attributes": {
+        "description": "Attributes being discussed about Sabian products",
+        "categories": [
+            {"value": "sound_quality", "description": "Tone, character, audio qualities"},
+            {"value": "durability", "description": "Build quality, longevity"},
+            {"value": "price", "description": "Cost and value"},
+            {"value": "playability", "description": "Feel, response"},
+            {"value": "aesthetics", "description": "Appearance, finish"},
+            {"value": "volume", "description": "Loudness, projection"},
+            {"value": "sustain", "description": "How long sound lasts"},
+            {"value": "versatility", "description": "Range of applications"}
+        ]
+    },
+    "analysis_notes_guidelines": {
+        "description": "Keep to 1-2 sentences. Focus on Sabian-specific insights not captured by other fields."
+    }
+}

processing_brand_sentiment/config_files/brand_config.json

@@ -0,0 +1,111 @@
+{
+    "brand": {
+        "name": "Sabian",
+        "description": "Sabian is a Canadian manufacturer of cymbals founded in 1981",
+        "products": [
+            "HHX",
+            "AAX",
+            "Artisan",
+            "FRX",
+            "Omni",
+            "Chopper",
+            "Stratus",
+            "XSR",
+            "B8X",
+            "SBR"
+        ],
+        "product_aliases": {
+            "b8": "B8X",
+            "sbrs": "SBR",
+            "hhxs": "HHX",
+            "aaxs": "AAX",
+            "hhx's": "HHX",
+            "aax's": "AAX"
+        },
+        "product_descriptions": {
+            "HHX": "Hand Hammered Xtreme - Professional series with dark, complex tones",
+            "AAX": "Bright, cutting cymbals for modern music",
+            "Artisan": "Premium hand-crafted cymbals with unique character",
+            "FRX": "Frequency Reduced Xtreme - Lower volume cymbals",
+            "Omni": "Multi-purpose cymbals for various playing styles",
+            "Chopper": "Effect cymbals with unique sound",
+            "Stratus": "Dark, complex sounds for jazz and fusion",
+            "XSR": "Entry-level professional cymbals",
+            "B8X": "Bronze entry-level cymbals",
+            "SBR": "Entry-level brass cymbals"
+        },
+        "competitor_products_warning": {
+            "description": "Products that belong to competitors - DO NOT attribute to Sabian",
+            "paiste_products": ["2002", "signature", "sound edge", "formula 602", "giant beat", "pst", "rude", "masters", "traditionals", "twenty", "dark energy"],
+            "zildjian_products": ["k custom", "a custom", "k zildjian", "a zildjian", "s family", "i family", "l80", "kerope", "constantinople", "k sweet"],
+            "meinl_products": ["byzance", "pure alloy", "hcs", "classics custom", "mb20", "mb10", "soundcaster"],
+            "dream_products": ["bliss", "contact", "energy", "dark matter", "vintage bliss", "eclipse"],
+            "istanbul_products": ["agop", "xist", "traditional", "sultan", "mehmet"]
+        },
+        "competitors": [
+            {
+                "name": "Zildjian",
+                "aliases": ["zildjian", "zil", "z custom", "a custom", "k custom", "k zildjian", "a zildjian"]
+            },
+            {
+                "name": "Meinl",
+                "aliases": ["meinl", "byzance", "classics"]
+            },
+            {
+                "name": "Paiste",
+                "aliases": ["paiste", "2002", "signature", "formula 602", "sound edge"]
+            },
+            {
+                "name": "Dream Cymbals",
+                "aliases": ["dream", "dream cymbals", "bliss"]
+            },
+            {
+                "name": "Istanbul Agop",
+                "aliases": ["istanbul", "agop", "istanbul agop", "istanbul mehmet"]
+            },
+            {
+                "name": "Bosphorus",
+                "aliases": ["bosphorus"]
+            }
+        ]
+    },
+    "relevance_keywords": {
+        "primary": {
+            "description": "Keywords that definitively indicate Sabian content",
+            "keywords": ["sabian", "hhx", "aax", "artisan", "frx", "omni", "chopper", "stratus", "xsr", "b8x", "sbr"]
+        },
+        "contextual": {
+            "description": "Ambiguous keywords that need context verification",
+            "keywords": ["b8"]
+        },
+        "cymbal_context": {
+            "description": "Keywords that provide cymbal-related context for disambiguation",
+            "keywords": ["cymbal", "cymbals", "crash", "ride", "hi-hat", "hihat", "hi hat", "splash", "china", "bell", "stack", "effects"]
+        }
+    },
+    "preprocessing": {
+        "min_length_for_language_detection": 50,
+        "default_language_for_short_text": "English",
+        "always_process_if_primary_keyword": true,
+        "min_content_length": 3
+    },
+    "filter_conditions": {
+        "exclude_access_levels": ["team", "house-coach"],
+        "exclude_post_states": ["deleted", "spam"],
+        "require_content_length_min": 3
+    },
+    "data_sources": {
+        "forums": {
+            "table": "SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_FORUM_POSTS",
+            "description": "Forum posts mentioning Sabian and their products",
+            "sql_query_file": "database/sql/fetch_forum_posts.sql",
+            "platform": "musora_forums"
+        },
+        "comments": {
+            "table": "SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS",
+            "description": "Social media comments potentially related to Sabian brand",
+            "sql_query_file": "database/sql/fetch_comments.sql",
+            "platform_column": "PLATFORM"
+        }
+    }
+}

processing_brand_sentiment/config_files/workflow_config.json

@@ -0,0 +1,60 @@
+{
+    "llm": {
+        "default_model": "gpt-5-nano",
+        "default_temperature": 0.2,
+        "max_retries": 3,
+        "timeout": 60
+    },
+    "agents": {
+        "preprocessor": {
+            "name": "PreprocessorAgent",
+            "description": "Deterministic agent for HTML parsing, text cleaning, language detection",
+            "model": "gpt-5-nano",
+            "temperature": 0.0,
+            "uses_llm": false
+        },
+        "relevance_validator": {
+            "name": "RelevanceValidatorAgent",
+            "description": "Lightweight LLM for disambiguation of ambiguous terms (HH, AA)",
+            "model": "gpt-5-nano",
+            "temperature": 0.0,
+            "max_retries": 2
+        },
+        "brand_analyzer": {
+            "name": "SabianAnalyzerAgent",
+            "description": "Comprehensive brand analysis for Sabian products",
+            "model": "gpt-5-nano",
+            "temperature": 0.2,
+            "max_retries": 3
+        }
+    },
+    "workflow": {
+        "parallel_processing": {
+            "enabled": true,
+            "worker_calculation": "CPU count - 2, max 5 workers",
+            "max_workers": 5,
+            "min_batch_size": 20,
+            "max_batch_size": 500
+        },
+        "thread_context": {
+            "enabled": true,
+            "include_thread_title": true,
+            "include_first_post": true
+        }
+    },
+    "output": {
+        "table_name": "SABIAN_BRAND_ANALYSIS",
+        "database": "SOCIAL_MEDIA_DB",
+        "schema": "ML_FEATURES"
+    },
+    "comments_output": {
+        "table_name": "SABIAN_BRAND_ANALYSIS_COMMENTS",
+        "database": "SOCIAL_MEDIA_DB",
+        "schema": "ML_FEATURES"
+    },
+    "logging": {
+        "level": "INFO",
+        "log_directory": "logs",
+        "log_file_prefix": "brand_sentiment_processing"
+    }
+}

processing_brand_sentiment/database/__init__.py

@@ -0,0 +1,8 @@
+"""
+Database module for brand sentiment analysis.
+Contains Snowflake connection handler and SQL query utilities.
+"""
+
+from .snowflake_connection import SnowFlakeConn
+
+__all__ = ['SnowFlakeConn']

processing_brand_sentiment/database/snowflake_connection.py

@@ -0,0 +1,240 @@
+"""
+Snowflake connection handler for brand sentiment analysis.
+Provides methods for reading data, executing queries, and storing results.
+"""
+
+import os
+from snowflake.snowpark import Session
+from dotenv import load_dotenv
+import logging
+import pandas as pd
+from typing import Optional, List, Any
+
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+
+
+class SnowFlakeConn:
+    """
+    Handles Snowflake database connections and operations for brand sentiment analysis.
+    """
+
+    def __init__(self):
+        """Initialize Snowflake connection."""
+        self.session = self.connect_to_snowflake()
+
+    def connect_to_snowflake(self) -> Session:
+        """
+        Create a connection to Snowflake using environment variables.
+
+        Returns:
+            Snowflake Session object
+        """
+        conn = dict(
+            user=self.get_credential("SNOWFLAKE_USER"),
+            password=self.get_credential("SNOWFLAKE_PASSWORD"),
+            account=self.get_credential("SNOWFLAKE_ACCOUNT"),
+            role=self.get_credential("SNOWFLAKE_ROLE"),
+            database=self.get_credential("SNOWFLAKE_DATABASE"),
+            warehouse=self.get_credential("SNOWFLAKE_WAREHOUSE"),
+            schema=self.get_credential("SNOWFLAKE_SCHEMA"),
+        )
+
+        session = Session.builder.configs(conn).create()
+        logger.info("Successfully connected to Snowflake")
+        return session
+
+    def get_credential(self, key: str) -> str:
+        """
+        Get credential from environment variables.
+
+        Args:
+            key: Environment variable name
+
+        Returns:
+            Credential value
+        """
+        return os.getenv(key)
+
+    def run_read_query(self, query: str, description: str = "data") -> pd.DataFrame:
+        """
+        Execute a SQL query that fetches data.
+
+        Args:
+            query: SQL query string
+            description: Description of what data is being fetched
+
+        Returns:
+            Pandas DataFrame containing query results
+        """
+        try:
+            dataframe = self.session.sql(query).to_pandas()
+            dataframe.columns = dataframe.columns.str.lower()
+            logger.info(f"Successfully read {len(dataframe)} rows for {description}")
+            return dataframe
+        except Exception as e:
+            logger.error(f"Error reading {description}: {e}")
+            raise
+
+    def store_df_to_snowflake(
+        self,
+        table_name: str,
+        dataframe: pd.DataFrame,
+        database: str = "SOCIAL_MEDIA_DB",
+        schema: str = "ML_FEATURES",
+        overwrite: bool = False
+    ) -> None:
+        """
+        Store a DataFrame to Snowflake.
+
+        Args:
+            table_name: Target table name
+            dataframe: DataFrame to store
+            database: Target database
+            schema: Target schema
+            overwrite: If True, overwrite existing data; if False, append
+        """
+        try:
+            self.session.use_database(database)
+            self.session.use_schema(schema)
+
+            dataframe = dataframe.reset_index(drop=True)
+            dataframe.columns = dataframe.columns.str.upper()
+
+            self.session.write_pandas(
+                df=dataframe,
+                table_name=table_name.strip().upper(),
+                auto_create_table=True,
+                overwrite=overwrite,
+                use_logical_type=True
+            )
+            logger.info(f"Successfully stored {len(dataframe)} rows to {table_name}")
+
+        except Exception as e:
+            logger.error(f"Error storing data to {table_name}: {e}")
+            raise
+
+    def execute_sql_file(self, file_path: str) -> Optional[List[Any]]:
+        """
+        Execute SQL queries from a file.
+
+        Args:
+            file_path: Path to SQL file
+
+        Returns:
+            Query result or None for DDL/DML
+        """
+        try:
+            with open(file_path, 'r', encoding='utf-8') as file:
+                sql_content = file.read()
+
+            result = self.session.sql(sql_content).collect()
+            logger.info(f"Successfully executed SQL from {file_path}")
+            return result
+        except Exception as e:
+            logger.error(f"Error executing SQL file {file_path}: {e}")
+            return None
+
+    def execute_query(self, query: str, description: str = "query") -> Optional[List[Any]]:
+        """
+        Execute a SQL query and return results.
+
+        Args:
+            query: SQL query string
+            description: Description of the query for logging
+
+        Returns:
+            Query results
+        """
+        try:
+            result = self.session.sql(query).collect()
+            logger.info(f"Successfully executed {description}")
+            return result
+        except Exception as e:
+            logger.error(f"Error executing {description}: {e}")
+            return None
+
+    def fetch_forum_posts_with_context(
+        self,
+        sql_file_path: str,
+        limit: Optional[int] = None
+    ) -> pd.DataFrame:
+        """
+        Fetch forum posts with thread context from SQL file.
+
+        Args:
+            sql_file_path: Path to the SQL query file
+            limit: Optional limit on number of posts to fetch
+
+        Returns:
+            DataFrame containing forum posts with context
+        """
+        try:
+            with open(sql_file_path, 'r', encoding='utf-8') as f:
+                query = f.read()
+
+            # Add limit if specified
+            if limit:
+                # Strip whitespace first, then semicolon, to handle Windows line endings
+                query = query.strip().rstrip(';') + f"\nLIMIT {limit};"
+
+            df = self.run_read_query(query, "forum posts with context")
+
+            # Validate required columns
+            required_cols = ['post_id', 'post_content', 'thread_id']
+            missing_cols = [col for col in required_cols if col not in df.columns]
+            if missing_cols:
+                logger.warning(f"Missing expected columns: {missing_cols}")
+
+            return df
+
+        except Exception as e:
+            logger.error(f"Error fetching forum posts: {e}")
+            raise
+
+    def fetch_comments(
+        self,
+        sql_file_path: str,
+        limit: Optional[int] = None
+    ) -> pd.DataFrame:
+        """
+        Fetch social media comments with context from SQL file.
+
+        Args:
+            sql_file_path: Path to the SQL query file
+            limit: Optional limit on number of comments to fetch
+
+        Returns:
+            DataFrame containing comments with context
+        """
+        try:
+            with open(sql_file_path, 'r', encoding='utf-8') as f:
+                query = f.read()
+
+            # Add limit if specified
+            if limit:
+                query = query.strip().rstrip(';') + f"\nLIMIT {limit};"
+
+            df = self.run_read_query(query, "social media comments with context")
+
+            # Validate required columns
+            required_cols = ['comment_sk', 'comment_id', 'comment_text', 'platform']
+            missing_cols = [col for col in required_cols if col not in df.columns]
+            if missing_cols:
+                logger.warning(f"Missing expected columns: {missing_cols}")
+
+            return df
+
+        except Exception as e:
+            logger.error(f"Error fetching comments: {e}")
+            raise
+
+    def close_connection(self) -> None:
+        """Close the Snowflake session."""
+        try:
+            self.session.close()
+            logger.info("Snowflake connection closed")
+        except Exception as e:
+            logger.error(f"Error closing connection: {e}")

processing_brand_sentiment/database/sql/create_comments_output_table.sql

@@ -0,0 +1,161 @@
+-- Create the output table for Sabian brand sentiment analysis on social media comments
+-- Stores processed comments with extracted brand intelligence
+-- Schema Version 4.0: Same analysis fields as forum table, different source identifiers
+
+CREATE TABLE IF NOT EXISTS SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS (
+    -- Source identifiers (comment-specific)
+    COMMENT_SK NUMBER(38,0),
+    COMMENT_ID VARCHAR(16777216),
+    ORIGINAL_TEXT VARCHAR(16777216),
+    PLATFORM VARCHAR(16777216),
+    COMMENT_TIMESTAMP TIMESTAMP_NTZ(9),
+    AUTHOR_NAME VARCHAR(16777216),
+    AUTHOR_ID VARCHAR(16777216),
+    CONTENT_SK NUMBER(38,0),
+    CONTENT_ID VARCHAR(16777216),
+    CONTENT_DESCRIPTION VARCHAR(16777216),
+    CHANNEL_SK NUMBER(38,0),
+    CHANNEL_NAME VARCHAR(16777216),
+    CHANNEL_DISPLAY_NAME VARCHAR(16777216),
+    PARENT_COMMENT_ID VARCHAR(16777216),
+    PARENT_COMMENT_TEXT VARCHAR(16777216),
+
+    -- Language detection
+    DETECTED_LANGUAGE VARCHAR(100),
+    LANGUAGE_CODE VARCHAR(10),
+    IS_ENGLISH BOOLEAN,
+
+    -- Relevance assessment
+    IS_RELEVANT BOOLEAN,
+    RELEVANCE_CONFIDENCE VARCHAR(20),
+    RELEVANCE_REASON VARCHAR(500),
+
+    -- Author classification
+    AUTHOR_ROLE VARCHAR(50),  -- current_owner, past_owner, potential_buyer, never_owned, unknown
+    SABIAN_MENTION_CONTEXT VARCHAR(50),  -- primary_focus, significant_mention, casual_mention, comparison_context
+
+    -- Sentiment analysis
+    SENTIMENT_LEVEL VARCHAR(20),
+    EMOTION_TYPE VARCHAR(50),
+    SENTIMENT_TARGET VARCHAR(50),
+    SENTIMENT_CONFIDENCE VARCHAR(20),
+
+    -- Product information (stored as JSON arrays)
+    PRODUCTS_MENTIONED VARCHAR(16777216),  -- JSON array: ["HHX", "AAX"]
+    PRODUCT_ATTRIBUTES VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+    PURCHASE_STAGE VARCHAR(50),
+
+    -- Competitive intelligence
+    COMPETITORS_MENTIONED VARCHAR(16777216),  -- JSON array: ["Zildjian", "Meinl"]
+    COMPETITOR_PRODUCTS_OWNED VARCHAR(16777216),  -- JSON array: competitor brands the AUTHOR owns
+    COMPARISON_TYPE VARCHAR(50),
+    COMPETITIVE_POSITIONING VARCHAR(500),
+    BRAND_SWITCHING VARCHAR(100),
+
+    -- Customer journey (AUTHOR PERSPECTIVE ONLY - null if giving advice to others)
+    INTENTS VARCHAR(16777216),  -- JSON array (multi-label): ["sharing_experience", "praising"]
+    DECISION_DRIVERS VARCHAR(16777216),  -- JSON array: ["sound_quality", "price"]
+    PAIN_POINTS VARCHAR(16777216),  -- JSON array: ["price_value", "availability"]
+    DELIGHT_FACTORS VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+
+    -- Analysis notes
+    ANALYSIS_NOTES VARCHAR(16777216),
+    SARCASM_DETECTED BOOLEAN,
+
+    -- Validation results
+    VALIDATION_PASSED BOOLEAN,
+    VALIDATION_ERRORS VARCHAR(16777216),  -- JSON array of error messages
+    VALIDATION_WARNINGS VARCHAR(16777216),  -- JSON array of warning messages
+    VALIDATION_FLAGS VARCHAR(16777216),  -- JSON array of anomaly flags
+
+    -- Processing metadata
+    PROCESSING_SUCCESS BOOLEAN,
+    PROCESSING_STATUS VARCHAR(50),  -- completed, completed_with_flags, validation_failed, workflow_error
+    PROCESSING_ERRORS VARCHAR(16777216),
+    PROCESSED_AT TIMESTAMP_LTZ(9) DEFAULT CURRENT_TIMESTAMP(),
+    WORKFLOW_VERSION VARCHAR(20) DEFAULT '4.0'
+)
+COMMENT = 'Brand sentiment analysis results for Sabian social media comments. Schema v4.0: 4-agent pipeline with extraction/analysis separation and validation.';
+
+-- Create indexes for common query patterns
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_SK ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(COMMENT_SK);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_PLATFORM ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(PLATFORM);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_IS_RELEVANT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(IS_RELEVANT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_SENTIMENT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(SENTIMENT_LEVEL);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_PROCESSED_AT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(PROCESSED_AT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_AUTHOR_ROLE ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(AUTHOR_ROLE);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_MENTION_CTX ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(SABIAN_MENTION_CONTEXT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_STATUS ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(PROCESSING_STATUS);
+
+-- Create view for relevant comments only
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMMENTS_RELEVANT_ANALYSIS AS
+SELECT *
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE;
+
+-- Create view for comments needing review (flagged by validator)
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMMENTS_FLAGGED AS
+SELECT
+    COMMENT_SK,
+    COMMENT_ID,
+    PLATFORM,
+    ORIGINAL_TEXT,
+    IS_RELEVANT,
+    RELEVANCE_CONFIDENCE,
+    RELEVANCE_REASON,
+    PRODUCTS_MENTIONED,
+    SABIAN_MENTION_CONTEXT,
+    SENTIMENT_LEVEL,
+    VALIDATION_FLAGS,
+    VALIDATION_WARNINGS,
+    PROCESSING_STATUS
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS
+WHERE PROCESSING_STATUS = 'completed_with_flags'
+   OR VALIDATION_PASSED = FALSE
+ORDER BY PROCESSED_AT DESC;
+
+-- Create view for sentiment distribution
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMMENTS_SENTIMENT_DISTRIBUTION AS
+SELECT
+    PLATFORM,
+    SENTIMENT_LEVEL,
+    EMOTION_TYPE,
+    SENTIMENT_TARGET,
+    COUNT(*) AS COMMENT_COUNT,
+    COUNT(CASE WHEN SARCASM_DETECTED = TRUE THEN 1 END) AS SARCASM_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+GROUP BY PLATFORM, SENTIMENT_LEVEL, EMOTION_TYPE, SENTIMENT_TARGET
+ORDER BY COMMENT_COUNT DESC;
+
+-- Create view for product mentions summary
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMMENTS_PRODUCT_MENTIONS AS
+SELECT
+    PLATFORM,
+    TRIM(product.VALUE::STRING) AS PRODUCT,
+    SENTIMENT_LEVEL,
+    COUNT(*) AS MENTION_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('positive', 'very_positive') THEN 1 END) AS POSITIVE_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('negative', 'very_negative') THEN 1 END) AS NEGATIVE_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS,
+    LATERAL FLATTEN(INPUT => TRY_PARSE_JSON(PRODUCTS_MENTIONED)) AS product
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+  AND PRODUCTS_MENTIONED IS NOT NULL
+GROUP BY PLATFORM, TRIM(product.VALUE::STRING), SENTIMENT_LEVEL
+ORDER BY MENTION_COUNT DESC;
+
+-- Create view for validation summary
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMMENTS_VALIDATION_SUMMARY AS
+SELECT
+    PLATFORM,
+    PROCESSING_STATUS,
+    VALIDATION_PASSED,
+    COUNT(*) AS COMMENT_COUNT,
+    COUNT(CASE WHEN IS_RELEVANT = TRUE THEN 1 END) AS RELEVANT_COUNT,
+    COUNT(CASE WHEN IS_RELEVANT = FALSE THEN 1 END) AS NOT_RELEVANT_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS
+GROUP BY PLATFORM, PROCESSING_STATUS, VALIDATION_PASSED
+ORDER BY COMMENT_COUNT DESC;

processing_brand_sentiment/database/sql/create_output_table.sql

@@ -0,0 +1,250 @@
+-- Create the output table for Sabian brand sentiment analysis
+-- Stores processed forum posts with extracted brand intelligence
+-- Schema Version 4.0: Added THREAD_CONTEXT_SUMMARY, validation fields, and processing status
+
+CREATE TABLE IF NOT EXISTS SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS (
+    -- Source identifiers
+    POST_ID NUMBER(38,0) PRIMARY KEY,
+    THREAD_ID NUMBER(38,0),
+    POST_AUTHOR_ID NUMBER(38,0),
+
+    -- Original and processed content
+    ORIGINAL_CONTENT VARCHAR(16777216),
+    CLEANED_CONTENT VARCHAR(16777216),
+    QUOTED_CONTENT VARCHAR(16777216),
+    THREAD_CONTEXT VARCHAR(16777216),  -- Raw thread context (legacy)
+    THREAD_CONTEXT_SUMMARY VARCHAR(4096),  -- NEW v4.0: Summarized thread context for analysis
+
+    -- Thread metadata
+    THREAD_TITLE VARCHAR(16777216),
+    THREAD_FIRST_POST VARCHAR(16777216),
+
+    -- Timestamps
+    POST_CREATED_AT TIMESTAMP_LTZ(9),
+    THREAD_STARTED_AT TIMESTAMP_LTZ(9),
+
+    -- Category information
+    CATEGORY_TITLE VARCHAR(16777216),
+    CATEGORY_TOPIC VARCHAR(16777216),
+
+    -- Language detection
+    DETECTED_LANGUAGE VARCHAR(100),
+    LANGUAGE_CODE VARCHAR(10),
+    IS_ENGLISH BOOLEAN,
+
+    -- Relevance assessment
+    IS_RELEVANT BOOLEAN,
+    RELEVANCE_CONFIDENCE VARCHAR(20),
+    RELEVANCE_REASON VARCHAR(500),
+
+    -- Author classification
+    AUTHOR_ROLE VARCHAR(50),  -- current_owner, past_owner, potential_buyer, never_owned, unknown
+    SABIAN_MENTION_CONTEXT VARCHAR(50),  -- primary_focus, significant_mention, casual_mention, comparison_context
+
+    -- Sentiment analysis
+    SENTIMENT_LEVEL VARCHAR(20),
+    EMOTION_TYPE VARCHAR(50),
+    SENTIMENT_TARGET VARCHAR(50),
+    SENTIMENT_CONFIDENCE VARCHAR(20),
+
+    -- Product information (stored as JSON arrays)
+    PRODUCTS_MENTIONED VARCHAR(16777216),  -- JSON array: ["HHX", "AAX"]
+    PRODUCT_ATTRIBUTES VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+    PURCHASE_STAGE VARCHAR(50),
+
+    -- Competitive intelligence
+    COMPETITORS_MENTIONED VARCHAR(16777216),  -- JSON array: ["Zildjian", "Meinl"]
+    COMPETITOR_PRODUCTS_OWNED VARCHAR(16777216),  -- JSON array: competitor brands the AUTHOR owns
+    COMPARISON_TYPE VARCHAR(50),
+    COMPETITIVE_POSITIONING VARCHAR(500),
+    BRAND_SWITCHING VARCHAR(100),
+
+    -- Customer journey (AUTHOR PERSPECTIVE ONLY - null if giving advice to others)
+    INTENTS VARCHAR(16777216),  -- JSON array (multi-label): ["sharing_experience", "praising"]
+    DECISION_DRIVERS VARCHAR(16777216),  -- JSON array: ["sound_quality", "price"]
+    PAIN_POINTS VARCHAR(16777216),  -- JSON array: ["price_value", "availability"]
+    DELIGHT_FACTORS VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+
+    -- Analysis notes
+    ANALYSIS_NOTES VARCHAR(16777216),
+    SARCASM_DETECTED BOOLEAN,
+
+    -- Validation results (NEW v4.0)
+    VALIDATION_PASSED BOOLEAN,
+    VALIDATION_ERRORS VARCHAR(16777216),  -- JSON array of error messages
+    VALIDATION_WARNINGS VARCHAR(16777216),  -- JSON array of warning messages
+    VALIDATION_FLAGS VARCHAR(16777216),  -- JSON array of anomaly flags (e.g., "sarcasm_detected", "low_confidence_relevant")
+
+    -- Platform identifier
+    PLATFORM VARCHAR(50) DEFAULT 'musora_forums',
+
+    -- Processing metadata
+    PROCESSING_SUCCESS BOOLEAN,
+    PROCESSING_STATUS VARCHAR(50),  -- NEW v4.0: completed, completed_with_flags, validation_failed, workflow_error
+    PROCESSING_ERRORS VARCHAR(16777216),
+    PROCESSED_AT TIMESTAMP_LTZ(9) DEFAULT CURRENT_TIMESTAMP(),
+    WORKFLOW_VERSION VARCHAR(20) DEFAULT '4.0'
+)
+COMMENT = 'Brand sentiment analysis results for Sabian forum posts. Schema v4.0: 4-agent pipeline with extraction/analysis separation, thread context summarization, and validation.';
+
+-- Create indexes for common query patterns
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_THREAD_ID ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(THREAD_ID);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_IS_RELEVANT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(IS_RELEVANT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_SENTIMENT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(SENTIMENT_LEVEL);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_PROCESSED_AT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(PROCESSED_AT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_AUTHOR_ROLE ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(AUTHOR_ROLE);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_MENTION_CONTEXT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(SABIAN_MENTION_CONTEXT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_PROCESSING_STATUS ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(PROCESSING_STATUS);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_VALIDATION_FLAGS ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(VALIDATION_PASSED);
+
+-- Create view for relevant posts only
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_RELEVANT_ANALYSIS AS
+SELECT *
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE;
+
+-- Create view for posts needing review (flagged by validator)
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_FLAGGED_POSTS AS
+SELECT
+    POST_ID,
+    THREAD_ID,
+    CLEANED_CONTENT,
+    THREAD_CONTEXT_SUMMARY,
+    IS_RELEVANT,
+    RELEVANCE_CONFIDENCE,
+    RELEVANCE_REASON,
+    PRODUCTS_MENTIONED,
+    SABIAN_MENTION_CONTEXT,
+    SENTIMENT_LEVEL,
+    VALIDATION_FLAGS,
+    VALIDATION_WARNINGS,
+    PROCESSING_STATUS
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+WHERE PROCESSING_STATUS = 'completed_with_flags'
+   OR VALIDATION_PASSED = FALSE
+ORDER BY PROCESSED_AT DESC;
+
+-- Create view for sentiment distribution
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_SENTIMENT_DISTRIBUTION AS
+SELECT
+    SENTIMENT_LEVEL,
+    EMOTION_TYPE,
+    SENTIMENT_TARGET,
+    COUNT(*) AS POST_COUNT,
+    COUNT(CASE WHEN SARCASM_DETECTED = TRUE THEN 1 END) AS SARCASM_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+GROUP BY SENTIMENT_LEVEL, EMOTION_TYPE, SENTIMENT_TARGET
+ORDER BY POST_COUNT DESC;
+
+-- Create view for product mentions summary
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_PRODUCT_MENTIONS AS
+SELECT
+    TRIM(product.VALUE::STRING) AS PRODUCT,
+    SENTIMENT_LEVEL,
+    COUNT(*) AS MENTION_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('positive', 'very_positive') THEN 1 END) AS POSITIVE_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('negative', 'very_negative') THEN 1 END) AS NEGATIVE_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS,
+    LATERAL FLATTEN(INPUT => TRY_PARSE_JSON(PRODUCTS_MENTIONED)) AS product
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+  AND PRODUCTS_MENTIONED IS NOT NULL
+GROUP BY TRIM(product.VALUE::STRING), SENTIMENT_LEVEL
+ORDER BY MENTION_COUNT DESC;
+
+-- Create view for competitor analysis
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMPETITOR_ANALYSIS AS
+SELECT
+    TRIM(competitor.VALUE::STRING) AS COMPETITOR,
+    COMPARISON_TYPE,
+    BRAND_SWITCHING,
+    COUNT(*) AS MENTION_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('positive', 'very_positive') THEN 1 END) AS POSITIVE_SENTIMENT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('negative', 'very_negative') THEN 1 END) AS NEGATIVE_SENTIMENT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS,
+    LATERAL FLATTEN(INPUT => TRY_PARSE_JSON(COMPETITORS_MENTIONED)) AS competitor
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+  AND COMPETITORS_MENTIONED IS NOT NULL
+GROUP BY TRIM(competitor.VALUE::STRING), COMPARISON_TYPE, BRAND_SWITCHING
+ORDER BY MENTION_COUNT DESC;
+
+-- Create view for pain points analysis
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_PAIN_POINTS AS
+SELECT
+    TRIM(pain_point.VALUE::STRING) AS PAIN_POINT,
+    COUNT(*) AS OCCURRENCE_COUNT,
+    ARRAY_AGG(DISTINCT SENTIMENT_LEVEL) AS SENTIMENT_LEVELS
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS,
+    LATERAL FLATTEN(INPUT => TRY_PARSE_JSON(PAIN_POINTS)) AS pain_point
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+  AND PAIN_POINTS IS NOT NULL
+GROUP BY TRIM(pain_point.VALUE::STRING)
+ORDER BY OCCURRENCE_COUNT DESC;
+
+-- Create view for author role analysis
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_AUTHOR_ROLES AS
+SELECT
+    AUTHOR_ROLE,
+    SABIAN_MENTION_CONTEXT,
+    COUNT(*) AS POST_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('positive', 'very_positive') THEN 1 END) AS POSITIVE_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('negative', 'very_negative') THEN 1 END) AS NEGATIVE_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL = 'neutral' THEN 1 END) AS NEUTRAL_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+GROUP BY AUTHOR_ROLE, SABIAN_MENTION_CONTEXT
+ORDER BY POST_COUNT DESC;
+
+-- Create view for competitor ownership analysis
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMPETITOR_OWNERSHIP AS
+SELECT
+    TRIM(competitor.VALUE::STRING) AS COMPETITOR_OWNED,
+    AUTHOR_ROLE,
+    COUNT(*) AS AUTHOR_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('positive', 'very_positive') THEN 1 END) AS POSITIVE_TOWARD_SABIAN,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('negative', 'very_negative') THEN 1 END) AS NEGATIVE_TOWARD_SABIAN
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS,
+    LATERAL FLATTEN(INPUT => TRY_PARSE_JSON(COMPETITOR_PRODUCTS_OWNED)) AS competitor
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+  AND COMPETITOR_PRODUCTS_OWNED IS NOT NULL
+GROUP BY TRIM(competitor.VALUE::STRING), AUTHOR_ROLE
+ORDER BY AUTHOR_COUNT DESC;
+
+-- Create view for mention context by sentiment
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_MENTION_DEPTH AS
+SELECT
+    SABIAN_MENTION_CONTEXT,
+    SENTIMENT_LEVEL,
+    COUNT(*) AS POST_COUNT,
+    AVG(CASE
+        WHEN SENTIMENT_LEVEL = 'very_positive' THEN 2
+        WHEN SENTIMENT_LEVEL = 'positive' THEN 1
+        WHEN SENTIMENT_LEVEL = 'neutral' THEN 0
+        WHEN SENTIMENT_LEVEL = 'negative' THEN -1
+        WHEN SENTIMENT_LEVEL = 'very_negative' THEN -2
+        ELSE 0
+    END) AS AVG_SENTIMENT_SCORE
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+GROUP BY SABIAN_MENTION_CONTEXT, SENTIMENT_LEVEL
+ORDER BY SABIAN_MENTION_CONTEXT, POST_COUNT DESC;
+
+-- Create view for validation flags analysis (NEW v4.0)
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_VALIDATION_SUMMARY AS
+SELECT
+    PROCESSING_STATUS,
+    VALIDATION_PASSED,
+    COUNT(*) AS POST_COUNT,
+    COUNT(CASE WHEN IS_RELEVANT = TRUE THEN 1 END) AS RELEVANT_COUNT,
+    COUNT(CASE WHEN IS_RELEVANT = FALSE THEN 1 END) AS NOT_RELEVANT_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+GROUP BY PROCESSING_STATUS, VALIDATION_PASSED
+ORDER BY POST_COUNT DESC;

processing_brand_sentiment/database/sql/fetch_comments.sql

@@ -0,0 +1,82 @@
+-- Query to fetch social media comments with context for brand sentiment analysis
+-- Source: SOCIAL_MEDIA_DB.brand_sentiment.SABIAN_comments (same structure as CORE.FACT_COMMENTS)
+-- Includes: comment content, parent comment text, content metadata, channel info
+-- Excludes: official accounts, already-processed comments, empty comments
+
+SELECT
+    -- Comment identifiers
+    fc.COMMENT_SK,
+    fc.COMMENT_ID,
+    fc.PLATFORM,
+    fc.MESSAGE AS COMMENT_TEXT,
+    fc.CREATED_TIME AS COMMENT_TIMESTAMP,
+    fc.AUTHOR_NAME,
+    fc.AUTHOR_ID,
+    fc.LIKE_COUNT,
+    fc.PARENT_COMMENT_ID,
+    fc.REPLIES_COUNT,
+    fc.COMMENT_LENGTH,
+    fc.IS_ACTIVE AS COMMENT_IS_ACTIVE,
+
+    -- Parent comment information (self-join to get parent comment text)
+    parent_fc.MESSAGE AS PARENT_COMMENT_TEXT,
+
+    -- Content information
+    dc.CONTENT_SK,
+    dc.CONTENT_ID,
+    dc.CONTENT_TYPE,
+    dc.MESSAGE AS CONTENT_DESCRIPTION,
+    dc.TITLE AS CONTENT_TITLE,
+    dc.PERMALINK_URL,
+    dc.CREATED_TIME AS CONTENT_TIMESTAMP,
+
+    -- Channel information
+    dch.CHANNEL_SK,
+    dch.CHANNEL_NAME,
+    dch.CHANNEL_DISPLAY_NAME
+
+FROM
+    SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS fc
+
+-- Left join to get parent comment text if it exists
+LEFT JOIN
+    SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS parent_fc
+    ON fc.PARENT_COMMENT_ID = parent_fc.COMMENT_ID
+    AND fc.PLATFORM = parent_fc.PLATFORM
+
+INNER JOIN
+    SOCIAL_MEDIA_DB.CORE.DIM_CONTENT dc
+    ON fc.CONTENT_SK = dc.CONTENT_SK
+
+INNER JOIN
+    SOCIAL_MEDIA_DB.CORE.DIM_CHANNEL dch
+    ON dc.CHANNEL_NAME = dch.CHANNEL_NAME
+    AND dc.PLATFORM = dch.PLATFORM
+
+-- Left join with output table to exclude already-processed comments
+LEFT JOIN
+    SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS sba
+    ON fc.COMMENT_SK = sba.COMMENT_SK
+
+WHERE
+    -- Active records only
+    fc.IS_ACTIVE = TRUE
+    AND dc.IS_ACTIVE = TRUE
+    AND dch.IS_ACTIVE = TRUE
+
+    -- Exclude official accounts
+    AND (fc.AUTHOR_NAME IS NULL OR fc.AUTHOR_NAME NOT IN (
+        'Musora', 'Drumeo', 'Pianote',
+        '@PianoteOfficial', '@DrumeoOfficial', '@MusoraOfficial'
+    ))
+
+    -- Exclude already-processed comments
+    AND sba.COMMENT_SK IS NULL
+
+    -- Ensure comment has content
+    AND fc.MESSAGE IS NOT NULL
+    AND TRIM(fc.MESSAGE) != ''
+    AND LENGTH(TRIM(fc.MESSAGE)) > 0
+
+ORDER BY
+    fc.CREATED_TIME DESC;

processing_brand_sentiment/database/sql/fetch_forum_posts.sql

@@ -0,0 +1,106 @@
+-- Query to fetch forum posts with thread context for brand sentiment analysis
+-- Includes: post content, thread context (title, first post), parent relationships
+-- Excludes: team/house-coach posts, already-processed posts, deleted posts
+
+WITH thread_first_posts AS (
+    -- Get the first post (by creation date) for each thread to use as context
+    -- Using ROW_NUMBER for reliable first post identification
+    SELECT
+        THREAD_ID,
+        POST_CONTENT AS FIRST_POST_CONTENT,
+        POST_AUTHOR_ID AS FIRST_POST_AUTHOR_ID,
+        POST_CREATED_AT AS FIRST_POST_CREATED_AT
+    FROM (
+        SELECT
+            THREAD_ID,
+            POST_CONTENT,
+            POST_AUTHOR_ID,
+            POST_CREATED_AT,
+            ROW_NUMBER() OVER (PARTITION BY THREAD_ID ORDER BY POST_CREATED_AT ASC) AS rn
+        FROM SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_FORUM_POSTS
+        WHERE POST_CONTENT IS NOT NULL
+          AND TRIM(POST_CONTENT) != ''
+    ) ranked
+    WHERE rn = 1
+)
+
+SELECT
+    -- Post identifiers
+    fp.POST_ID,
+    fp.POST_AUTHOR_ID,
+    fp.THREAD_ID,
+
+    -- Post content (may contain HTML with quoted parent)
+    fp.POST_CONTENT,
+
+    -- Post timestamps
+    fp.POST_CREATED_AT,
+    fp.POST_EDITED_ON,
+    fp.POST_PUBLISHED_ON,
+    fp.POST_STATE,
+
+    -- Parent/Child relationships (for context)
+    fp.PROMPTING_POST_ID,
+    fp.PARENT_ID,
+    fp.PARENT_CONTENT,
+    fp.PARENT_AUTHOR_ID,
+    fp.PARENT_CREATED_AT,
+    fp.CHILD_ID,
+    fp.CHILD_CONTENT,
+
+    -- Thread context
+    fp.THREAD_TITLE,
+    fp.THREAD_SLUG,
+    fp.THREAD_STATE,
+    fp.THREAD_LOCKED,
+    fp.THREAD_PINNED,
+    fp.THREAD_POST_COUNT,
+    fp.THREAD_PUBLISHED_ON,
+
+    -- First post of the thread (for context)
+    tfp.FIRST_POST_CONTENT AS THREAD_FIRST_POST,
+    tfp.FIRST_POST_CREATED_AT AS THREAD_STARTED_AT,
+
+    -- Category information
+    fp.CATEGORY_ID,
+    fp.CATEGORY_BRAND,
+    fp.CATEGORY_DESCRIPTION,
+    fp.CATEGORY_TITLE,
+    fp.CATEGORY_TOPIC,
+    fp.CATEGORY_SLUG,
+
+    -- Access levels (for filtering)
+    fp.POST_AUTHOR_ACCESS_LEVEL,
+    fp.PARENT_AUTHOR_ACCESS_LEVEL,
+    fp.CHILD_AUTHOR_ACCESS_LEVEL
+
+FROM
+    SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_FORUM_POSTS fp
+
+-- Join to get thread's first post for context
+LEFT JOIN
+    thread_first_posts tfp ON fp.THREAD_ID = tfp.THREAD_ID
+
+-- Left join with output table to exclude already-processed posts
+LEFT JOIN
+    SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS sba
+    ON fp.POST_ID = sba.POST_ID
+
+WHERE
+    -- Exclude team and house-coach posts (internal comments)
+    (fp.POST_AUTHOR_ACCESS_LEVEL IS NULL OR fp.POST_AUTHOR_ACCESS_LEVEL NOT IN ('team', 'house-coach'))
+
+    -- Exclude deleted posts
+    AND (fp.POST_STATE IS NULL OR fp.POST_STATE != 'deleted')
+    AND fp.POST_DELETED_AT IS NULL
+
+    -- Exclude already-processed posts
+    AND sba.POST_ID IS NULL
+
+    -- Ensure post has content
+    AND fp.POST_CONTENT IS NOT NULL
+    AND TRIM(fp.POST_CONTENT) != ''
+    AND LENGTH(TRIM(fp.POST_CONTENT)) > 0
+
+ORDER BY
+    fp.POST_CREATED_AT DESC;

processing_brand_sentiment/database/sql/init_comments_output_table.sql

@@ -0,0 +1,78 @@
+-- Initialize empty output table for Sabian brand sentiment analysis on social media comments
+-- Run this script BEFORE the first processing run to create the table structure
+-- This prevents "table not found" errors when the fetch query tries to check for already-processed comments
+
+CREATE TABLE IF NOT EXISTS SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS (
+    -- Source identifiers (comment-specific)
+    COMMENT_SK NUMBER(38,0),
+    COMMENT_ID VARCHAR(16777216),
+    ORIGINAL_TEXT VARCHAR(16777216),
+    PLATFORM VARCHAR(16777216),
+    COMMENT_TIMESTAMP TIMESTAMP_NTZ(9),
+    AUTHOR_NAME VARCHAR(16777216),
+    AUTHOR_ID VARCHAR(16777216),
+    CONTENT_SK NUMBER(38,0),
+    CONTENT_ID VARCHAR(16777216),
+    CONTENT_DESCRIPTION VARCHAR(16777216),
+    CHANNEL_SK NUMBER(38,0),
+    CHANNEL_NAME VARCHAR(16777216),
+    CHANNEL_DISPLAY_NAME VARCHAR(16777216),
+    PARENT_COMMENT_ID VARCHAR(16777216),
+    PARENT_COMMENT_TEXT VARCHAR(16777216),
+
+    -- Language detection
+    DETECTED_LANGUAGE VARCHAR(100),
+    LANGUAGE_CODE VARCHAR(10),
+    IS_ENGLISH BOOLEAN,
+
+    -- Relevance assessment
+    IS_RELEVANT BOOLEAN,
+    RELEVANCE_CONFIDENCE VARCHAR(20),
+    RELEVANCE_REASON VARCHAR(500),
+
+    -- Author classification
+    AUTHOR_ROLE VARCHAR(50),  -- current_owner, past_owner, potential_buyer, never_owned, unknown
+    SABIAN_MENTION_CONTEXT VARCHAR(50),  -- primary_focus, significant_mention, casual_mention, comparison_context
+
+    -- Sentiment analysis
+    SENTIMENT_LEVEL VARCHAR(20),
+    EMOTION_TYPE VARCHAR(50),
+    SENTIMENT_TARGET VARCHAR(50),
+    SENTIMENT_CONFIDENCE VARCHAR(20),
+
+    -- Product information (stored as JSON arrays)
+    PRODUCTS_MENTIONED VARCHAR(16777216),  -- JSON array: ["HHX", "AAX"]
+    PRODUCT_ATTRIBUTES VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+    PURCHASE_STAGE VARCHAR(50),
+
+    -- Competitive intelligence
+    COMPETITORS_MENTIONED VARCHAR(16777216),  -- JSON array: ["Zildjian", "Meinl"]
+    COMPETITOR_PRODUCTS_OWNED VARCHAR(16777216),  -- JSON array: competitor brands the AUTHOR owns
+    COMPARISON_TYPE VARCHAR(50),
+    COMPETITIVE_POSITIONING VARCHAR(500),
+    BRAND_SWITCHING VARCHAR(100),
+
+    -- Customer journey (AUTHOR PERSPECTIVE ONLY - null if giving advice to others)
+    INTENTS VARCHAR(16777216),  -- JSON array (multi-label): ["sharing_experience", "praising"]
+    DECISION_DRIVERS VARCHAR(16777216),  -- JSON array: ["sound_quality", "price"]
+    PAIN_POINTS VARCHAR(16777216),  -- JSON array: ["price_value", "availability"]
+    DELIGHT_FACTORS VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+
+    -- Analysis notes
+    ANALYSIS_NOTES VARCHAR(16777216),
+    SARCASM_DETECTED BOOLEAN,
+
+    -- Validation results
+    VALIDATION_PASSED BOOLEAN,
+    VALIDATION_ERRORS VARCHAR(16777216),  -- JSON array of error messages
+    VALIDATION_WARNINGS VARCHAR(16777216),  -- JSON array of warning messages
+    VALIDATION_FLAGS VARCHAR(16777216),  -- JSON array of anomaly flags
+
+    -- Processing metadata
+    PROCESSING_SUCCESS BOOLEAN,
+    PROCESSING_STATUS VARCHAR(50),  -- completed, completed_with_flags, validation_failed, workflow_error
+    PROCESSING_ERRORS VARCHAR(16777216),
+    PROCESSED_AT TIMESTAMP_LTZ(9) DEFAULT CURRENT_TIMESTAMP(),
+    WORKFLOW_VERSION VARCHAR(20) DEFAULT '4.0'
+)
+COMMENT = 'Brand sentiment analysis results for Sabian social media comments. Schema v4.0: 4-agent pipeline with extraction/analysis separation and validation.';

processing_brand_sentiment/database/sql/init_output_table.sql

@@ -0,0 +1,89 @@
+-- Initialize empty output table for Sabian brand sentiment analysis
+-- Run this script BEFORE the first processing run to create the table structure
+-- This prevents "table not found" errors when the fetch query tries to check for already-processed posts
+-- Schema Version 4.0: Added THREAD_CONTEXT_SUMMARY, validation fields, and processing status
+
+CREATE TABLE IF NOT EXISTS SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS (
+    -- Source identifiers
+    POST_ID NUMBER(38,0) PRIMARY KEY,
+    THREAD_ID NUMBER(38,0),
+    POST_AUTHOR_ID NUMBER(38,0),
+
+    -- Original and processed content
+    ORIGINAL_CONTENT VARCHAR(16777216),
+    CLEANED_CONTENT VARCHAR(16777216),
+    QUOTED_CONTENT VARCHAR(16777216),
+    THREAD_CONTEXT VARCHAR(16777216),  -- Raw thread context (legacy)
+    THREAD_CONTEXT_SUMMARY VARCHAR(4096),  -- NEW v4.0: Summarized thread context
+
+    -- Thread metadata
+    THREAD_TITLE VARCHAR(16777216),
+    THREAD_FIRST_POST VARCHAR(16777216),
+
+    -- Timestamps
+    POST_CREATED_AT TIMESTAMP_LTZ(9),
+    THREAD_STARTED_AT TIMESTAMP_LTZ(9),
+
+    -- Category information
+    CATEGORY_TITLE VARCHAR(16777216),
+    CATEGORY_TOPIC VARCHAR(16777216),
+
+    -- Language detection
+    DETECTED_LANGUAGE VARCHAR(100),
+    LANGUAGE_CODE VARCHAR(10),
+    IS_ENGLISH BOOLEAN,
+
+    -- Relevance assessment
+    IS_RELEVANT BOOLEAN,
+    RELEVANCE_CONFIDENCE VARCHAR(20),
+    RELEVANCE_REASON VARCHAR(500),
+
+    -- Author classification
+    AUTHOR_ROLE VARCHAR(50),  -- current_owner, past_owner, potential_buyer, never_owned, unknown
+    SABIAN_MENTION_CONTEXT VARCHAR(50),  -- primary_focus, significant_mention, casual_mention, comparison_context
+
+    -- Sentiment analysis
+    SENTIMENT_LEVEL VARCHAR(20),
+    EMOTION_TYPE VARCHAR(50),
+    SENTIMENT_TARGET VARCHAR(50),
+    SENTIMENT_CONFIDENCE VARCHAR(20),
+
+    -- Product information (stored as JSON arrays)
+    PRODUCTS_MENTIONED VARCHAR(16777216),
+    PRODUCT_ATTRIBUTES VARCHAR(16777216),
+
+    -- Competitive intelligence
+    COMPETITORS_MENTIONED VARCHAR(16777216),
+    COMPETITOR_PRODUCTS_OWNED VARCHAR(16777216),  -- JSON array: competitor brands the AUTHOR owns
+    COMPARISON_TYPE VARCHAR(50),
+    COMPETITIVE_POSITIONING VARCHAR(500),
+    BRAND_SWITCHING VARCHAR(100),
+
+    -- Customer journey (AUTHOR PERSPECTIVE ONLY - null if giving advice to others)
+    INTENTS VARCHAR(16777216),  -- Multi-label: seeking_information, providing_information, sharing_experience, comparing, praising, criticizing, buying_selling, general_discussion
+    PURCHASE_STAGE VARCHAR(50),  -- AUTHOR's own stage only
+    DECISION_DRIVERS VARCHAR(16777216),  -- AUTHOR's own decision drivers only
+    PAIN_POINTS VARCHAR(16777216),  -- AUTHOR's negative feedback aspects (uses feedback_aspects categories)
+    DELIGHT_FACTORS VARCHAR(16777216),  -- AUTHOR's positive feedback aspects (uses feedback_aspects categories)
+
+    -- Analysis notes
+    ANALYSIS_NOTES VARCHAR(16777216),
+    SARCASM_DETECTED BOOLEAN,
+
+    -- Validation results (NEW v4.0)
+    VALIDATION_PASSED BOOLEAN,
+    VALIDATION_ERRORS VARCHAR(16777216),  -- JSON array of error messages
+    VALIDATION_WARNINGS VARCHAR(16777216),  -- JSON array of warning messages
+    VALIDATION_FLAGS VARCHAR(16777216),  -- JSON array of anomaly flags
+
+    -- Platform identifier
+    PLATFORM VARCHAR(50) DEFAULT 'musora_forums',
+
+    -- Processing metadata
+    PROCESSING_SUCCESS BOOLEAN,
+    PROCESSING_STATUS VARCHAR(50),  -- NEW v4.0: completed, completed_with_flags, validation_failed, workflow_error
+    PROCESSING_ERRORS VARCHAR(16777216),
+    PROCESSED_AT TIMESTAMP_LTZ(9) DEFAULT CURRENT_TIMESTAMP(),
+    WORKFLOW_VERSION VARCHAR(20) DEFAULT '4.0'
+)
+COMMENT = 'Brand sentiment analysis results for Sabian forum posts. Schema v4.0: Added thread_context_summary, validation fields, and processing status.';

processing_brand_sentiment/main.py

@@ -0,0 +1,1088 @@
+"""
+Main execution script for brand sentiment analysis workflow.
+Orchestrates data fetching, processing, and storage using an agentic workflow.
+Supports parallel processing with multiprocessing for improved performance.
+Supports multiple data sources: forums, social media comments, or both.
+"""
+
+import json
+import os
+import logging
+import argparse
+from datetime import datetime
+import pandas as pd
+from dotenv import load_dotenv
+from multiprocessing import Pool, cpu_count
+import traceback
+from typing import Dict, Any, List
+
+from database.snowflake_connection import SnowFlakeConn
+from workflow.orchestrator import BrandAnalysisWorkflow
+from workflow.comment_orchestrator import CommentAnalysisWorkflow
+
+# Get the directory where this script is located
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Load environment variables
+load_dotenv(os.path.join(SCRIPT_DIR, '.env'))
+
+# Ensure logs directory exists
+LOGS_DIR = os.path.join(SCRIPT_DIR, 'logs')
+os.makedirs(LOGS_DIR, exist_ok=True)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler(
+            os.path.join(LOGS_DIR, f'brand_sentiment_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log')
+        ),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+
+# ============================================================
+# Configuration Loading
+# ============================================================
+
+def load_configs(config_dir: str = None) -> Dict[str, Dict]:
+    """
+    Load all configuration files.
+
+    Args:
+        config_dir: Directory containing config files
+
+    Returns:
+        Dictionary with all configurations
+    """
+    if config_dir is None:
+        config_dir = os.path.join(SCRIPT_DIR, 'config_files')
+
+    configs = {}
+
+    # Load workflow config
+    with open(os.path.join(config_dir, 'workflow_config.json'), 'r') as f:
+        configs['workflow'] = json.load(f)
+
+    # Load brand config
+    with open(os.path.join(config_dir, 'brand_config.json'), 'r') as f:
+        configs['brand'] = json.load(f)
+
+    # Load analysis categories
+    with open(os.path.join(config_dir, 'analysis_categories.json'), 'r') as f:
+        configs['categories'] = json.load(f)
+
+    return configs
+
+
+# ============================================================
+# Batch Processing Utilities
+# ============================================================
+
+def calculate_optimal_batch_size(
+    total_posts: int,
+    num_workers: int,
+    min_batch: int = 20,
+    max_batch: int = 500
+) -> int:
+    """
+    Calculate optimal batch size based on total posts and workers.
+
+    Args:
+        total_posts: Total number of posts to process
+        num_workers: Number of parallel workers
+        min_batch: Minimum batch size
+        max_batch: Maximum batch size
+
+    Returns:
+        Optimal batch size
+    """
+    if total_posts <= min_batch:
+        return total_posts
+
+    batch_size = total_posts // num_workers
+    batch_size = max(min_batch, min(max_batch, batch_size))
+
+    return batch_size
+
+
+def safe_to_json(value: Any) -> Any:
+    """
+    Safely convert a value to JSON string.
+    Handles None, NaN, lists, and already-string values.
+
+    Args:
+        value: Value to convert
+
+    Returns:
+        JSON string if list, None if null, original value otherwise
+    """
+    # Handle None and NaN
+    if value is None or (isinstance(value, float) and pd.isna(value)):
+        return None
+    # Handle lists - convert to JSON
+    if isinstance(value, list):
+        return json.dumps(value) if value else None
+    # Handle already-string values
+    if isinstance(value, str):
+        return value if value else None
+    # Return as-is for other types
+    return value
+
+
+def safe_json_list_length(value: Any) -> int:
+    """
+    Safely get the length of a JSON array string.
+    Handles None, NaN, empty strings, and invalid JSON.
+
+    Args:
+        value: Value to parse (expected JSON string of array)
+
+    Returns:
+        Length of the array, or 0 if invalid/empty
+    """
+    # Handle None and NaN
+    if value is None or (isinstance(value, float) and pd.isna(value)):
+        return 0
+    # Handle non-string values
+    if not isinstance(value, str):
+        return 0
+    # Handle empty strings
+    if not value or value == '[]' or value == 'null':
+        return 0
+    # Try to parse JSON
+    try:
+        parsed = json.loads(value)
+        return len(parsed) if isinstance(parsed, list) else 0
+    except (json.JSONDecodeError, TypeError):
+        return 0
+
+
+def calculate_batch_stats(df: pd.DataFrame) -> Dict[str, int]:
+    """
+    Calculate statistics from batch results.
+    Handles null values safely for all fields.
+
+    Args:
+        df: DataFrame with processed results
+
+    Returns:
+        Dictionary with statistics
+    """
+    stats = {
+        'relevant_count': 0,
+        'not_relevant_count': 0,
+        'products_mentioned_count': 0,
+        'competitors_mentioned_count': 0,
+        'positive_sentiment_count': 0,
+        'negative_sentiment_count': 0,
+        # Author role stats
+        'current_owner_count': 0,
+        'potential_buyer_count': 0,
+        'primary_focus_count': 0
+    }
+
+    # Handle empty dataframe
+    if df.empty:
+        return stats
+
+    # Count relevant/not relevant posts
+    if 'IS_RELEVANT' in df.columns:
+        relevant_col = df['IS_RELEVANT']
+        non_null_mask = relevant_col.notna()
+        if non_null_mask.any():
+            stats['relevant_count'] = int(relevant_col[non_null_mask].astype(bool).sum())
+            stats['not_relevant_count'] = int((~relevant_col[non_null_mask].astype(bool)).sum())
+
+    # Count product mentions using safe helper
+    if 'PRODUCTS_MENTIONED' in df.columns:
+        stats['products_mentioned_count'] = int(
+            df['PRODUCTS_MENTIONED'].apply(safe_json_list_length).sum()
+        )
+
+    # Count competitor mentions using safe helper
+    if 'COMPETITORS_MENTIONED' in df.columns:
+        stats['competitors_mentioned_count'] = int(
+            df['COMPETITORS_MENTIONED'].apply(safe_json_list_length).sum()
+        )
+
+    # Count sentiment distribution
+    if 'SENTIMENT_LEVEL' in df.columns:
+        sentiment_values = df['SENTIMENT_LEVEL'].dropna()
+        if not sentiment_values.empty:
+            stats['positive_sentiment_count'] = int(
+                sentiment_values.isin(['positive', 'very_positive']).sum()
+            )
+            stats['negative_sentiment_count'] = int(
+                sentiment_values.isin(['negative', 'very_negative']).sum()
+            )
+
+    # Count author roles
+    if 'AUTHOR_ROLE' in df.columns:
+        author_roles = df['AUTHOR_ROLE'].dropna()
+        if not author_roles.empty:
+            stats['current_owner_count'] = int((author_roles == 'current_owner').sum())
+            stats['potential_buyer_count'] = int((author_roles == 'potential_buyer').sum())
+
+    # Count mention context
+    if 'SABIAN_MENTION_CONTEXT' in df.columns:
+        mention_context = df['SABIAN_MENTION_CONTEXT'].dropna()
+        if not mention_context.empty:
+            stats['primary_focus_count'] = int((mention_context == 'primary_focus').sum())
+
+    return stats
+
+
+def aggregate_results(results: List[Dict[str, Any]]) -> Dict[str, Any]:
+    """
+    Aggregate results from multiple batches.
+
+    Args:
+        results: List of batch result dictionaries
+
+    Returns:
+        Aggregated statistics dictionary
+    """
+    aggregated = {
+        'total_processed': sum(r.get('total_processed', 0) for r in results),
+        'total_stored': sum(r.get('total_stored', 0) for r in results),
+        'failed_count': sum(r.get('failed_count', 0) for r in results),
+        'relevant_count': sum(r.get('relevant_count', 0) for r in results),
+        'not_relevant_count': sum(r.get('not_relevant_count', 0) for r in results),
+        'products_mentioned_count': sum(r.get('products_mentioned_count', 0) for r in results),
+        'competitors_mentioned_count': sum(r.get('competitors_mentioned_count', 0) for r in results),
+        'positive_sentiment_count': sum(r.get('positive_sentiment_count', 0) for r in results),
+        'negative_sentiment_count': sum(r.get('negative_sentiment_count', 0) for r in results),
+        'current_owner_count': sum(r.get('current_owner_count', 0) for r in results),
+        'potential_buyer_count': sum(r.get('potential_buyer_count', 0) for r in results),
+        'primary_focus_count': sum(r.get('primary_focus_count', 0) for r in results),
+        'failed_batches': sum(1 for r in results if not r.get('success', False))
+    }
+
+    # Log failed batches
+    failed_batches = [r for r in results if not r.get('success', False)]
+    if failed_batches:
+        logger.error(f"{len(failed_batches)} batch(es) failed:")
+        for fb in failed_batches:
+            logger.error(f"  Batch {fb.get('batch_num')}: {fb.get('error')}")
+
+    return aggregated
+
+
+# ============================================================
+# Forum Processing (existing functionality)
+# ============================================================
+
+# Columns that should be converted from lists to JSON strings
+FORUM_JSON_ARRAY_COLUMNS = [
+    'products_mentioned', 'product_attributes', 'competitors_mentioned',
+    'competitor_products_owned', 'intents', 'decision_drivers',
+    'pain_points', 'delight_factors', 'processing_errors', 'relevance_keywords_found',
+    'validation_errors', 'validation_warnings', 'validation_flags'
+]
+
+# Column mapping from forum workflow state to output table
+FORUM_COLUMN_MAPPING = {
+    'post_id': 'POST_ID',
+    'thread_id': 'THREAD_ID',
+    'post_author_id': 'POST_AUTHOR_ID',
+    'original_content': 'ORIGINAL_CONTENT',
+    'cleaned_content': 'CLEANED_CONTENT',
+    'quoted_content': 'QUOTED_CONTENT',
+    'raw_thread_context': 'THREAD_CONTEXT',
+    'thread_context_summary': 'THREAD_CONTEXT_SUMMARY',
+    'thread_title': 'THREAD_TITLE',
+    'thread_first_post': 'THREAD_FIRST_POST',
+    'post_created_at': 'POST_CREATED_AT',
+    'thread_started_at': 'THREAD_STARTED_AT',
+    'category_title': 'CATEGORY_TITLE',
+    'category_topic': 'CATEGORY_TOPIC',
+    'detected_language': 'DETECTED_LANGUAGE',
+    'language_code': 'LANGUAGE_CODE',
+    'is_english': 'IS_ENGLISH',
+    'is_relevant': 'IS_RELEVANT',
+    'relevance_confidence': 'RELEVANCE_CONFIDENCE',
+    'relevance_reason': 'RELEVANCE_REASON',
+    'author_role': 'AUTHOR_ROLE',
+    'sabian_mention_context': 'SABIAN_MENTION_CONTEXT',
+    'sentiment_level': 'SENTIMENT_LEVEL',
+    'emotion_type': 'EMOTION_TYPE',
+    'sentiment_target': 'SENTIMENT_TARGET',
+    'sentiment_confidence': 'SENTIMENT_CONFIDENCE',
+    'products_mentioned': 'PRODUCTS_MENTIONED',
+    'product_attributes': 'PRODUCT_ATTRIBUTES',
+    'competitors_mentioned': 'COMPETITORS_MENTIONED',
+    'competitor_products_owned': 'COMPETITOR_PRODUCTS_OWNED',
+    'comparison_type': 'COMPARISON_TYPE',
+    'competitive_positioning': 'COMPETITIVE_POSITIONING',
+    'brand_switching': 'BRAND_SWITCHING',
+    'intents': 'INTENTS',
+    'purchase_stage': 'PURCHASE_STAGE',
+    'decision_drivers': 'DECISION_DRIVERS',
+    'pain_points': 'PAIN_POINTS',
+    'delight_factors': 'DELIGHT_FACTORS',
+    'analysis_notes': 'ANALYSIS_NOTES',
+    'sarcasm_detected': 'SARCASM_DETECTED',
+    'validation_passed': 'VALIDATION_PASSED',
+    'validation_errors': 'VALIDATION_ERRORS',
+    'validation_warnings': 'VALIDATION_WARNINGS',
+    'validation_flags': 'VALIDATION_FLAGS',
+    'success': 'PROCESSING_SUCCESS',
+    'processing_status': 'PROCESSING_STATUS',
+    'processing_errors': 'PROCESSING_ERRORS'
+}
+
+
+def prepare_forum_output_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Prepare forum output DataFrame with proper column mapping.
+
+    Args:
+        df: DataFrame with processing results
+
+    Returns:
+        DataFrame ready for Snowflake storage
+    """
+    output_df = pd.DataFrame()
+
+    for source_col, target_col in FORUM_COLUMN_MAPPING.items():
+        if source_col in df.columns:
+            value = df[source_col].copy()
+            if source_col in FORUM_JSON_ARRAY_COLUMNS:
+                value = value.apply(safe_to_json)
+            output_df[target_col] = value
+        else:
+            output_df[target_col] = None
+
+    # Add metadata
+    output_df['PLATFORM'] = 'musora_forums'
+    output_df['PROCESSED_AT'] = datetime.now()
+    output_df['WORKFLOW_VERSION'] = '4.0'
+
+    return output_df
+
+
+def process_forum_batch_worker(batch_data: tuple) -> Dict[str, Any]:
+    """
+    Worker function to process a single batch of forum posts.
+    Runs in a separate process.
+
+    Args:
+        batch_data: Tuple containing (batch_num, posts, configs, api_key, overwrite_first_batch, output_config)
+
+    Returns:
+        Dictionary with batch statistics
+    """
+    batch_num, posts, configs, api_key, overwrite_first_batch, output_config = batch_data
+
+    worker_logger = logging.getLogger(f"ForumWorker-{batch_num}")
+
+    try:
+        worker_logger.info(f"Forum Batch {batch_num}: Starting processing of {len(posts)} posts")
+
+        # Initialize Snowflake connection for this worker
+        snowflake = SnowFlakeConn()
+
+        # Initialize workflow for this worker
+        workflow = BrandAnalysisWorkflow(
+            workflow_config=configs['workflow'],
+            brand_config=configs['brand'],
+            analysis_categories=configs['categories'],
+            api_key=api_key
+        )
+
+        # Process posts
+        results = workflow.process_batch(posts)
+
+        # Convert to DataFrame
+        results_df = pd.DataFrame(results)
+
+        # Filter successful results
+        initial_count = len(results_df)
+        df_successful = results_df[results_df['success'] == True].copy()
+        failed_count = initial_count - len(df_successful)
+
+        worker_logger.info(f"Forum Batch {batch_num}: Processed {initial_count} posts, {len(df_successful)} successful")
+
+        # Prepare output DataFrame
+        output_df = prepare_forum_output_dataframe(df_successful)
+
+        # Store results
+        if len(output_df) > 0:
+            overwrite = overwrite_first_batch and batch_num == 1
+
+            snowflake.store_df_to_snowflake(
+                table_name=output_config['table_name'],
+                dataframe=output_df,
+                database=output_config['database'],
+                schema=output_config['schema'],
+                overwrite=overwrite
+            )
+
+            worker_logger.info(f"Forum Batch {batch_num}: Stored {len(output_df)} records to Snowflake")
+        else:
+            worker_logger.warning(f"Forum Batch {batch_num}: No successful records to store")
+
+        # Close connection
+        snowflake.close_connection()
+
+        # Calculate statistics
+        stats = calculate_batch_stats(output_df)
+        stats.update({
+            'batch_num': batch_num,
+            'success': True,
+            'total_processed': initial_count,
+            'total_stored': len(output_df),
+            'failed_count': failed_count,
+            'error': None
+        })
+
+        return stats
+
+    except Exception as e:
+        error_msg = f"Forum Batch {batch_num} failed: {str(e)}"
+        worker_logger.error(error_msg)
+        worker_logger.error(traceback.format_exc())
+
+        return {
+            'batch_num': batch_num,
+            'success': False,
+            'total_processed': len(posts),
+            'total_stored': 0,
+            'failed_count': len(posts),
+            'error': error_msg
+        }
+
+
+# ============================================================
+# Comment Processing (new functionality)
+# ============================================================
+
+# Columns that should be converted from lists to JSON strings (same analysis fields)
+COMMENT_JSON_ARRAY_COLUMNS = [
+    'products_mentioned', 'product_attributes', 'competitors_mentioned',
+    'competitor_products_owned', 'intents', 'decision_drivers',
+    'pain_points', 'delight_factors', 'processing_errors', 'relevance_keywords_found',
+    'validation_errors', 'validation_warnings', 'validation_flags'
+]
+
+# Column mapping from comment workflow state to output table
+COMMENT_COLUMN_MAPPING = {
+    # Comment-specific identifiers
+    'comment_sk': 'COMMENT_SK',
+    'comment_id': 'COMMENT_ID',
+    'original_text': 'ORIGINAL_TEXT',
+    'platform': 'PLATFORM',
+    'comment_timestamp': 'COMMENT_TIMESTAMP',
+    'author_name': 'AUTHOR_NAME',
+    'author_id': 'AUTHOR_ID',
+    'content_sk': 'CONTENT_SK',
+    'content_id': 'CONTENT_ID',
+    'content_description': 'CONTENT_DESCRIPTION',
+    'channel_sk': 'CHANNEL_SK',
+    'channel_name': 'CHANNEL_NAME',
+    'channel_display_name': 'CHANNEL_DISPLAY_NAME',
+    'parent_comment_id': 'PARENT_COMMENT_ID',
+    'parent_comment_text': 'PARENT_COMMENT_TEXT',
+    # Analysis fields (same as forums)
+    'detected_language': 'DETECTED_LANGUAGE',
+    'language_code': 'LANGUAGE_CODE',
+    'is_english': 'IS_ENGLISH',
+    'is_relevant': 'IS_RELEVANT',
+    'relevance_confidence': 'RELEVANCE_CONFIDENCE',
+    'relevance_reason': 'RELEVANCE_REASON',
+    'author_role': 'AUTHOR_ROLE',
+    'sabian_mention_context': 'SABIAN_MENTION_CONTEXT',
+    'sentiment_level': 'SENTIMENT_LEVEL',
+    'emotion_type': 'EMOTION_TYPE',
+    'sentiment_target': 'SENTIMENT_TARGET',
+    'sentiment_confidence': 'SENTIMENT_CONFIDENCE',
+    'products_mentioned': 'PRODUCTS_MENTIONED',
+    'product_attributes': 'PRODUCT_ATTRIBUTES',
+    'purchase_stage': 'PURCHASE_STAGE',
+    'competitors_mentioned': 'COMPETITORS_MENTIONED',
+    'competitor_products_owned': 'COMPETITOR_PRODUCTS_OWNED',
+    'comparison_type': 'COMPARISON_TYPE',
+    'competitive_positioning': 'COMPETITIVE_POSITIONING',
+    'brand_switching': 'BRAND_SWITCHING',
+    'intents': 'INTENTS',
+    'decision_drivers': 'DECISION_DRIVERS',
+    'pain_points': 'PAIN_POINTS',
+    'delight_factors': 'DELIGHT_FACTORS',
+    'analysis_notes': 'ANALYSIS_NOTES',
+    'sarcasm_detected': 'SARCASM_DETECTED',
+    'validation_passed': 'VALIDATION_PASSED',
+    'validation_errors': 'VALIDATION_ERRORS',
+    'validation_warnings': 'VALIDATION_WARNINGS',
+    'validation_flags': 'VALIDATION_FLAGS',
+    'success': 'PROCESSING_SUCCESS',
+    'processing_status': 'PROCESSING_STATUS',
+    'processing_errors': 'PROCESSING_ERRORS'
+}
+
+
+def prepare_comment_output_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Prepare comment output DataFrame with proper column mapping.
+
+    Args:
+        df: DataFrame with processing results
+
+    Returns:
+        DataFrame ready for Snowflake storage
+    """
+    output_df = pd.DataFrame()
+
+    for source_col, target_col in COMMENT_COLUMN_MAPPING.items():
+        if source_col in df.columns:
+            value = df[source_col].copy()
+            if source_col in COMMENT_JSON_ARRAY_COLUMNS:
+                value = value.apply(safe_to_json)
+            output_df[target_col] = value
+        else:
+            output_df[target_col] = None
+
+    # Add metadata
+    output_df['PROCESSED_AT'] = datetime.now()
+    output_df['WORKFLOW_VERSION'] = '4.0'
+
+    return output_df
+
+
+def process_comment_batch_worker(batch_data: tuple) -> Dict[str, Any]:
+    """
+    Worker function to process a single batch of social media comments.
+    Runs in a separate process.
+
+    Args:
+        batch_data: Tuple containing (batch_num, comments, configs, api_key, overwrite_first_batch, output_config)
+
+    Returns:
+        Dictionary with batch statistics
+    """
+    batch_num, comments, configs, api_key, overwrite_first_batch, output_config = batch_data
+
+    worker_logger = logging.getLogger(f"CommentWorker-{batch_num}")
+
+    try:
+        worker_logger.info(f"Comment Batch {batch_num}: Starting processing of {len(comments)} comments")
+
+        # Initialize Snowflake connection for this worker
+        snowflake = SnowFlakeConn()
+
+        # Initialize comment workflow for this worker
+        workflow = CommentAnalysisWorkflow(
+            workflow_config=configs['workflow'],
+            brand_config=configs['brand'],
+            analysis_categories=configs['categories'],
+            api_key=api_key
+        )
+
+        # Process comments
+        results = workflow.process_batch(comments)
+
+        # Convert to DataFrame
+        results_df = pd.DataFrame(results)
+
+        # Filter successful results
+        initial_count = len(results_df)
+        df_successful = results_df[results_df['success'] == True].copy()
+        failed_count = initial_count - len(df_successful)
+
+        worker_logger.info(f"Comment Batch {batch_num}: Processed {initial_count} comments, {len(df_successful)} successful")
+
+        # Prepare output DataFrame
+        output_df = prepare_comment_output_dataframe(df_successful)
+
+        # Store results
+        if len(output_df) > 0:
+            overwrite = overwrite_first_batch and batch_num == 1
+
+            snowflake.store_df_to_snowflake(
+                table_name=output_config['table_name'],
+                dataframe=output_df,
+                database=output_config['database'],
+                schema=output_config['schema'],
+                overwrite=overwrite
+            )
+
+            worker_logger.info(f"Comment Batch {batch_num}: Stored {len(output_df)} records to Snowflake")
+        else:
+            worker_logger.warning(f"Comment Batch {batch_num}: No successful records to store")
+
+        # Close connection
+        snowflake.close_connection()
+
+        # Calculate statistics
+        stats = calculate_batch_stats(output_df)
+        stats.update({
+            'batch_num': batch_num,
+            'success': True,
+            'total_processed': initial_count,
+            'total_stored': len(output_df),
+            'failed_count': failed_count,
+            'error': None
+        })
+
+        return stats
+
+    except Exception as e:
+        error_msg = f"Comment Batch {batch_num} failed: {str(e)}"
+        worker_logger.error(error_msg)
+        worker_logger.error(traceback.format_exc())
+
+        return {
+            'batch_num': batch_num,
+            'success': False,
+            'total_processed': len(comments),
+            'total_stored': 0,
+            'failed_count': len(comments),
+            'error': error_msg
+        }
+
+
+# ============================================================
+# Main Processor Class
+# ============================================================
+
+class BrandSentimentProcessor:
+    """
+    Main processor class that orchestrates the entire workflow.
+    Supports processing forums, social media comments, or both.
+    """
+
+    def __init__(self, config_dir: str = None):
+        """
+        Initialize the processor.
+
+        Args:
+            config_dir: Directory containing configuration files
+        """
+        # Load configurations
+        self.configs = load_configs(config_dir)
+
+        # Initialize Snowflake connection
+        self.snowflake = SnowFlakeConn()
+
+        # Get OpenAI API key
+        self.api_key = os.getenv("OPENAI_API_KEY")
+        if not self.api_key:
+            raise ValueError("OPENAI_API_KEY not found in environment variables")
+
+        # Get output configurations
+        self.forum_output_config = self.configs['workflow'].get('output', {
+            'table_name': 'SABIAN_BRAND_ANALYSIS',
+            'database': 'SOCIAL_MEDIA_DB',
+            'schema': 'ML_FEATURES'
+        })
+
+        self.comment_output_config = self.configs['workflow'].get('comments_output', {
+            'table_name': 'SABIAN_BRAND_ANALYSIS_COMMENTS',
+            'database': 'SOCIAL_MEDIA_DB',
+            'schema': 'ML_FEATURES'
+        })
+
+        logger.info("BrandSentimentProcessor initialized successfully")
+
+    def fetch_forum_posts(self, limit: int = None) -> pd.DataFrame:
+        """
+        Fetch forum posts from Snowflake.
+
+        Args:
+            limit: Optional limit on number of posts
+
+        Returns:
+            DataFrame containing post data
+        """
+        logger.info("Fetching forum posts...")
+
+        sql_path = os.path.join(SCRIPT_DIR, 'database', 'sql', 'fetch_forum_posts.sql')
+        df = self.snowflake.fetch_forum_posts_with_context(sql_path, limit)
+
+        logger.info(f"Fetched {len(df)} forum posts")
+        return df
+
+    def fetch_comments(self, limit: int = None) -> pd.DataFrame:
+        """
+        Fetch social media comments from Snowflake.
+
+        Args:
+            limit: Optional limit on number of comments
+
+        Returns:
+            DataFrame containing comment data
+        """
+        logger.info("Fetching social media comments...")
+
+        sql_path = os.path.join(SCRIPT_DIR, 'database', 'sql', 'fetch_comments.sql')
+        df = self.snowflake.fetch_comments(sql_path, limit)
+
+        logger.info(f"Fetched {len(df)} social media comments")
+        return df
+
+    def calculate_num_workers(self) -> int:
+        """
+        Calculate number of parallel workers.
+
+        Returns:
+            Number of workers
+        """
+        parallel_config = self.configs['workflow'].get('workflow', {}).get('parallel_processing', {})
+        max_workers = parallel_config.get('max_workers', 5)
+
+        num_cpus = cpu_count()
+        num_workers = max(1, min(max_workers, num_cpus - 2))
+
+        logger.info(f"Using {num_workers} parallel workers (CPU count: {num_cpus})")
+        return num_workers
+
+    # ---- Forum Processing ----
+
+    def process_forums_parallel(self, df: pd.DataFrame, overwrite: bool = False) -> Dict[str, Any]:
+        """
+        Process forum posts using parallel workers.
+
+        Args:
+            df: DataFrame containing posts
+            overwrite: Whether to overwrite existing table
+
+        Returns:
+            Dictionary with aggregated statistics
+        """
+        posts = df.to_dict('records')
+        total_posts = len(posts)
+
+        logger.info(f"Processing {total_posts} forum posts using parallel processing...")
+
+        num_workers = self.calculate_num_workers()
+
+        parallel_config = self.configs['workflow'].get('workflow', {}).get('parallel_processing', {})
+        min_batch = parallel_config.get('min_batch_size', 20)
+        max_batch = parallel_config.get('max_batch_size', 400)
+
+        batch_size = calculate_optimal_batch_size(total_posts, num_workers, min_batch, max_batch)
+        logger.info(f"Forum batch size: {batch_size}")
+
+        # Create batches
+        batches = []
+        for i in range(0, total_posts, batch_size):
+            batch = posts[i:i + batch_size]
+            batch_num = (i // batch_size) + 1
+            batches.append((batch_num, batch, self.configs, self.api_key, overwrite, self.forum_output_config))
+
+        total_batches = len(batches)
+        logger.info(f"Split into {total_batches} forum batches")
+
+        # Process in parallel
+        with Pool(processes=num_workers) as pool:
+            results = pool.map(process_forum_batch_worker, batches)
+
+        return aggregate_results(results)
+
+    def process_forums_sequential(self, df: pd.DataFrame, overwrite: bool = False) -> Dict[str, Any]:
+        """
+        Process forum posts sequentially (for debugging).
+
+        Args:
+            df: DataFrame containing posts
+            overwrite: Whether to overwrite existing table
+
+        Returns:
+            Dictionary with statistics
+        """
+        logger.info(f"Processing {len(df)} forum posts using sequential processing...")
+
+        posts = df.to_dict('records')
+        batch_data = (1, posts, self.configs, self.api_key, overwrite, self.forum_output_config)
+        result = process_forum_batch_worker(batch_data)
+
+        return {
+            'total_processed': result.get('total_processed', 0),
+            'total_stored': result.get('total_stored', 0),
+            'failed_count': result.get('failed_count', 0),
+            'relevant_count': result.get('relevant_count', 0),
+            'not_relevant_count': result.get('not_relevant_count', 0),
+            'products_mentioned_count': result.get('products_mentioned_count', 0),
+            'competitors_mentioned_count': result.get('competitors_mentioned_count', 0),
+            'positive_sentiment_count': result.get('positive_sentiment_count', 0),
+            'negative_sentiment_count': result.get('negative_sentiment_count', 0),
+            'current_owner_count': result.get('current_owner_count', 0),
+            'potential_buyer_count': result.get('potential_buyer_count', 0),
+            'primary_focus_count': result.get('primary_focus_count', 0),
+            'failed_batches': 0 if result.get('success', False) else 1
+        }
+
+    # ---- Comment Processing ----
+
+    def process_comments_parallel(self, df: pd.DataFrame, overwrite: bool = False) -> Dict[str, Any]:
+        """
+        Process social media comments using parallel workers.
+
+        Args:
+            df: DataFrame containing comments
+            overwrite: Whether to overwrite existing table
+
+        Returns:
+            Dictionary with aggregated statistics
+        """
+        comments = df.to_dict('records')
+        total_comments = len(comments)
+
+        logger.info(f"Processing {total_comments} comments using parallel processing...")
+
+        num_workers = self.calculate_num_workers()
+
+        parallel_config = self.configs['workflow'].get('workflow', {}).get('parallel_processing', {})
+        min_batch = parallel_config.get('min_batch_size', 20)
+        max_batch = parallel_config.get('max_batch_size', 400)
+
+        batch_size = calculate_optimal_batch_size(total_comments, num_workers, min_batch, max_batch)
+        logger.info(f"Comment batch size: {batch_size}")
+
+        # Create batches
+        batches = []
+        for i in range(0, total_comments, batch_size):
+            batch = comments[i:i + batch_size]
+            batch_num = (i // batch_size) + 1
+            batches.append((batch_num, batch, self.configs, self.api_key, overwrite, self.comment_output_config))
+
+        total_batches = len(batches)
+        logger.info(f"Split into {total_batches} comment batches")
+
+        # Process in parallel
+        with Pool(processes=num_workers) as pool:
+            results = pool.map(process_comment_batch_worker, batches)
+
+        return aggregate_results(results)
+
+    def process_comments_sequential(self, df: pd.DataFrame, overwrite: bool = False) -> Dict[str, Any]:
+        """
+        Process social media comments sequentially (for debugging).
+
+        Args:
+            df: DataFrame containing comments
+            overwrite: Whether to overwrite existing table
+
+        Returns:
+            Dictionary with statistics
+        """
+        logger.info(f"Processing {len(df)} comments using sequential processing...")
+
+        comments = df.to_dict('records')
+        batch_data = (1, comments, self.configs, self.api_key, overwrite, self.comment_output_config)
+        result = process_comment_batch_worker(batch_data)
+
+        return {
+            'total_processed': result.get('total_processed', 0),
+            'total_stored': result.get('total_stored', 0),
+            'failed_count': result.get('failed_count', 0),
+            'relevant_count': result.get('relevant_count', 0),
+            'not_relevant_count': result.get('not_relevant_count', 0),
+            'products_mentioned_count': result.get('products_mentioned_count', 0),
+            'competitors_mentioned_count': result.get('competitors_mentioned_count', 0),
+            'positive_sentiment_count': result.get('positive_sentiment_count', 0),
+            'negative_sentiment_count': result.get('negative_sentiment_count', 0),
+            'current_owner_count': result.get('current_owner_count', 0),
+            'potential_buyer_count': result.get('potential_buyer_count', 0),
+            'primary_focus_count': result.get('primary_focus_count', 0),
+            'failed_batches': 0 if result.get('success', False) else 1
+        }
+
+    # ---- Unified Processing ----
+
+    def _log_source_summary(self, source_name: str, stats: Dict[str, Any], processing_time: float) -> None:
+        """
+        Log processing summary for a data source.
+
+        Args:
+            source_name: Name of the data source
+            stats: Processing statistics
+            processing_time: Time taken in seconds
+        """
+        logger.info(f"  --- {source_name} ---")
+        logger.info(f"  Total processed: {stats.get('total_processed', 0)}")
+        logger.info(f"  Successfully stored: {stats.get('total_stored', 0)}")
+        logger.info(f"  Failed: {stats.get('failed_count', 0)}")
+        logger.info(f"  Relevant: {stats.get('relevant_count', 0)}")
+        logger.info(f"  Not relevant: {stats.get('not_relevant_count', 0)}")
+        logger.info(f"  Product mentions: {stats.get('products_mentioned_count', 0)}")
+        logger.info(f"  Competitor mentions: {stats.get('competitors_mentioned_count', 0)}")
+        logger.info(f"  Positive sentiment: {stats.get('positive_sentiment_count', 0)}")
+        logger.info(f"  Negative sentiment: {stats.get('negative_sentiment_count', 0)}")
+        logger.info(f"  Current owners: {stats.get('current_owner_count', 0)}")
+        logger.info(f"  Potential buyers: {stats.get('potential_buyer_count', 0)}")
+        logger.info(f"  Primary focus: {stats.get('primary_focus_count', 0)}")
+        if stats.get('failed_batches', 0) > 0:
+            logger.info(f"  Failed batches: {stats['failed_batches']}")
+        logger.info(f"  Processing time: {processing_time:.2f} seconds")
+        if stats.get('total_processed', 0) > 0:
+            logger.info(f"  Average per item: {processing_time / stats['total_processed']:.2f} seconds")
+
+    def run(
+        self,
+        limit: int = None,
+        overwrite: bool = False,
+        sequential: bool = False,
+        data_source: str = 'all'
+    ):
+        """
+        Run the complete processing pipeline.
+
+        Args:
+            limit: Optional limit on items to process per source
+            overwrite: Whether to overwrite existing table
+            sequential: Use sequential processing instead of parallel
+            data_source: Which data source to process ('forums', 'comments', 'all')
+        """
+        try:
+            logger.info("=" * 80)
+            logger.info("Starting Brand Sentiment Analysis Workflow")
+            logger.info(f"Brand: {self.configs['brand'].get('brand', {}).get('name', 'Unknown')}")
+            logger.info(f"Mode: {'SEQUENTIAL' if sequential else 'PARALLEL'}")
+            logger.info(f"Data source: {data_source}")
+            logger.info("=" * 80)
+
+            process_forums = data_source in ('forums', 'all')
+            process_comments = data_source in ('comments', 'all')
+
+            # Track results for summary
+            forum_stats = None
+            forum_time = 0.0
+            comment_stats = None
+            comment_time = 0.0
+
+            # ---- Process Forums ----
+            if process_forums:
+                logger.info("-" * 40)
+                logger.info("Processing FORUMS")
+                logger.info("-" * 40)
+
+                df_posts = self.fetch_forum_posts(limit)
+
+                if df_posts.empty:
+                    logger.warning("No forum posts to process")
+                else:
+                    start_time = datetime.now()
+
+                    if sequential:
+                        forum_stats = self.process_forums_sequential(df_posts, overwrite)
+                    else:
+                        forum_stats = self.process_forums_parallel(df_posts, overwrite)
+
+                    forum_time = (datetime.now() - start_time).total_seconds()
+
+            # ---- Process Comments ----
+            if process_comments:
+                logger.info("-" * 40)
+                logger.info("Processing SOCIAL MEDIA COMMENTS")
+                logger.info("-" * 40)
+
+                df_comments = self.fetch_comments(limit)
+
+                if df_comments.empty:
+                    logger.warning("No social media comments to process")
+                else:
+                    start_time = datetime.now()
+
+                    if sequential:
+                        comment_stats = self.process_comments_sequential(df_comments, overwrite)
+                    else:
+                        comment_stats = self.process_comments_parallel(df_comments, overwrite)
+
+                    comment_time = (datetime.now() - start_time).total_seconds()
+
+            # ---- Summary ----
+            logger.info("=" * 80)
+            logger.info("Processing Summary:")
+            logger.info(f"  Mode: {'Sequential' if sequential else 'Parallel'}")
+            logger.info(f"  Data source: {data_source}")
+
+            if forum_stats is not None:
+                self._log_source_summary("Forums", forum_stats, forum_time)
+
+            if comment_stats is not None:
+                self._log_source_summary("Social Media Comments", comment_stats, comment_time)
+
+            logger.info("=" * 80)
+
+        except Exception as e:
+            logger.error(f"Error in workflow execution: {str(e)}", exc_info=True)
+            raise
+
+        finally:
+            self.snowflake.close_connection()
+            logger.info("Snowflake connection closed")
+
+
+# ============================================================
+# Legacy compatibility - keep old function names working
+# ============================================================
+
+def prepare_output_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """Legacy wrapper for forum output preparation."""
+    return prepare_forum_output_dataframe(df)
+
+
+def process_batch_worker(batch_data: tuple) -> Dict[str, Any]:
+    """Legacy wrapper for forum batch worker."""
+    return process_forum_batch_worker(batch_data)
+
+
+# ============================================================
+# Main Entry Point
+# ============================================================
+
+def main():
+    """Main entry point."""
+    parser = argparse.ArgumentParser(
+        description="Brand Sentiment Analysis - Analyze forum posts and social media comments for brand intelligence"
+    )
+    parser.add_argument(
+        '--limit',
+        type=int,
+        default=None,
+        help='Limit number of items to process per source (default: all unprocessed)'
+    )
+    parser.add_argument(
+        '--overwrite',
+        action='store_true',
+        default=False,
+        help='Overwrite existing Snowflake table (default: append)'
+    )
+    parser.add_argument(
+        '--sequential',
+        action='store_true',
+        default=False,
+        help='Use sequential processing instead of parallel (for debugging)'
+    )
+    parser.add_argument(
+        '--config-dir',
+        type=str,
+        default=None,
+        help='Path to configuration directory (default: config_files/)'
+    )
+    parser.add_argument(
+        '--data-source',
+        type=str,
+        choices=['forums', 'comments', 'all'],
+        default='all',
+        help='Data source to process: forums, comments, or all (default: all)'
+    )
+
+    args = parser.parse_args()
+
+    # Initialize and run
+    processor = BrandSentimentProcessor(config_dir=args.config_dir)
+    processor.run(
+        limit=args.limit,
+        overwrite=args.overwrite,
+        sequential=args.sequential,
+        data_source=args.data_source
+    )
+
+
+if __name__ == "__main__":
+    main()

processing_brand_sentiment/utils/__init__.py

@@ -0,0 +1,8 @@
+"""
+Utilities module for brand sentiment analysis.
+Contains HTML parsing and other helper functions.
+"""
+
+from .html_parser import HTMLParser
+
+__all__ = ['HTMLParser']

processing_brand_sentiment/utils/html_parser.py

@@ -0,0 +1,253 @@
+"""
+HTML Parser utility for extracting content from forum posts.
+Handles the complex HTML structure where replies contain quoted parent content.
+"""
+
+import re
+import html
+from typing import Dict, Optional, Tuple
+from bs4 import BeautifulSoup
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class HTMLParser:
+    """
+    Parses HTML content from forum posts to extract actual reply content
+    and quoted parent content separately.
+    """
+
+    def __init__(self):
+        """Initialize the HTML parser."""
+        pass
+
+    def parse_post_content(self, html_content: str) -> Dict[str, Optional[str]]:
+        """
+        Parse HTML post content to extract reply and quoted content.
+
+        The forum posts have a structure where:
+        - <blockquote> contains the quoted parent post
+        - Content outside blockquote is the actual reply
+
+        Example input:
+        <blockquote><span class="post-id">125015</span>
+        <p class="quote-heading"><strong>JackO</strong><em> - Feb 3, 2015</em></p>
+        <br /><p>Parent content here...</p></blockquote>
+        <br /><p>Actual reply content here...</p>
+
+        Args:
+            html_content: Raw HTML content from POST_CONTENT field
+
+        Returns:
+            Dictionary with:
+            - reply_content: The actual reply text (cleaned)
+            - quoted_content: The quoted parent text (cleaned), if any
+            - quoted_author: Author of the quoted post, if any
+            - quoted_date: Date of the quoted post, if any
+            - has_quote: Boolean indicating if post contains a quote
+        """
+        if not html_content or not html_content.strip():
+            return {
+                "reply_content": "",
+                "quoted_content": None,
+                "quoted_author": None,
+                "quoted_date": None,
+                "has_quote": False
+            }
+
+        try:
+            soup = BeautifulSoup(html_content, 'html.parser')
+
+            # Extract quoted content from blockquotes
+            quoted_content = None
+            quoted_author = None
+            quoted_date = None
+            has_quote = False
+
+            blockquotes = soup.find_all('blockquote')
+
+            if blockquotes:
+                has_quote = True
+                quote_parts = []
+
+                for blockquote in blockquotes:
+                    # Extract quote heading info (author and date)
+                    quote_heading = blockquote.find('p', class_='quote-heading')
+                    if quote_heading:
+                        author_tag = quote_heading.find('strong')
+                        if author_tag:
+                            quoted_author = author_tag.get_text(strip=True)
+
+                        date_tag = quote_heading.find('em')
+                        if date_tag:
+                            quoted_date = date_tag.get_text(strip=True).lstrip(' - ')
+
+                    # Get the quote text content (excluding heading)
+                    # Remove the heading first to get just the content
+                    if quote_heading:
+                        quote_heading.decompose()
+
+                    # Remove post-id spans
+                    for post_id_span in blockquote.find_all('span', class_='post-id'):
+                        post_id_span.decompose()
+
+                    quote_text = self._clean_text(blockquote.get_text())
+                    if quote_text:
+                        quote_parts.append(quote_text)
+
+                    # Remove the blockquote from the soup to get remaining content
+                    blockquote.decompose()
+
+                quoted_content = " ".join(quote_parts) if quote_parts else None
+
+            # Get the remaining content (actual reply)
+            reply_content = self._clean_text(soup.get_text())
+
+            return {
+                "reply_content": reply_content,
+                "quoted_content": quoted_content,
+                "quoted_author": quoted_author,
+                "quoted_date": quoted_date,
+                "has_quote": has_quote
+            }
+
+        except Exception as e:
+            logger.warning(f"Error parsing HTML content: {e}")
+            # Fallback: try to extract text directly
+            return {
+                "reply_content": self._clean_text(self._strip_html_tags(html_content)),
+                "quoted_content": None,
+                "quoted_author": None,
+                "quoted_date": None,
+                "has_quote": False
+            }
+
+    def _clean_text(self, text: str) -> str:
+        """
+        Clean extracted text by removing extra whitespace and normalizing.
+
+        Args:
+            text: Raw text to clean
+
+        Returns:
+            Cleaned text
+        """
+        if not text:
+            return ""
+
+        # Decode HTML entities
+        text = html.unescape(text)
+
+        # Replace multiple whitespace with single space
+        text = re.sub(r'\s+', ' ', text)
+
+        # Strip leading/trailing whitespace
+        text = text.strip()
+
+        return text
+
+    def _strip_html_tags(self, html_content: str) -> str:
+        """
+        Fallback method to strip HTML tags if BeautifulSoup fails.
+
+        Args:
+            html_content: HTML content
+
+        Returns:
+            Text without HTML tags
+        """
+        # Remove HTML tags
+        clean = re.sub(r'<[^>]+>', ' ', html_content)
+        # Decode entities
+        clean = html.unescape(clean)
+        # Clean whitespace
+        clean = re.sub(r'\s+', ' ', clean)
+        return clean.strip()
+
+    def extract_plain_text(self, html_content: str) -> str:
+        """
+        Extract plain text from HTML content, preserving readability.
+
+        Args:
+            html_content: HTML content
+
+        Returns:
+            Plain text version
+        """
+        if not html_content:
+            return ""
+
+        try:
+            soup = BeautifulSoup(html_content, 'html.parser')
+
+            # Add newlines for block elements
+            for br in soup.find_all('br'):
+                br.replace_with('\n')
+            for p in soup.find_all('p'):
+                p.append('\n')
+
+            text = soup.get_text()
+            return self._clean_text(text)
+
+        except Exception as e:
+            logger.warning(f"Error extracting plain text: {e}")
+            return self._clean_text(self._strip_html_tags(html_content))
+
+    def build_thread_context(
+        self,
+        thread_title: Optional[str],
+        first_post_content: Optional[str],
+        category_title: Optional[str] = None,
+        category_topic: Optional[str] = None
+    ) -> str:
+        """
+        Build a context string from thread information.
+
+        Args:
+            thread_title: Title of the discussion thread
+            first_post_content: Content of the first post in the thread
+            category_title: Category title
+            category_topic: Category topic
+
+        Returns:
+            Formatted context string
+        """
+        context_parts = []
+
+        if category_title:
+            context_parts.append(f"Category: {category_title}")
+
+        if category_topic:
+            context_parts.append(f"Topic: {category_topic}")
+
+        if thread_title:
+            context_parts.append(f"Thread: {thread_title}")
+
+        if first_post_content:
+            # Parse and clean the first post content
+            parsed = self.parse_post_content(first_post_content)
+            first_post_text = parsed.get("reply_content", "")
+            if first_post_text:
+                # Truncate if too long
+                if len(first_post_text) > 500:
+                    first_post_text = first_post_text[:500] + "..."
+                context_parts.append(f"Original discussion: {first_post_text}")
+
+        return " | ".join(context_parts) if context_parts else ""
+
+    def is_empty_content(self, html_content: str) -> bool:
+        """
+        Check if HTML content is effectively empty.
+
+        Args:
+            html_content: HTML content to check
+
+        Returns:
+            True if content is empty or contains no meaningful text
+        """
+        if not html_content:
+            return True
+
+        text = self.extract_plain_text(html_content)
+        return len(text.strip()) == 0

processing_brand_sentiment/workflow/__init__.py

@@ -0,0 +1,10 @@
+"""
+Workflow module for brand sentiment analysis.
+Contains the LangGraph orchestrators and agent implementations.
+Supports both forum posts and social media comments.
+"""
+
+from .orchestrator import BrandAnalysisWorkflow
+from .comment_orchestrator import CommentAnalysisWorkflow
+
+__all__ = ['BrandAnalysisWorkflow', 'CommentAnalysisWorkflow']

processing_brand_sentiment/workflow/agents/__init__.py

@@ -0,0 +1,39 @@
+"""
+Agents module for brand sentiment analysis v4.0.
+
+Contains specialized agents for the 4-stage pipeline:
+1. ContentPreprocessorAgent - HTML parsing, cleaning, keyword detection (forums)
+   CommentPreprocessorAgent - Plain text cleaning, keyword detection (comments)
+2. SabianRelevanceExtractionAgent - Relevance + fact extraction
+3. SabianSentimentAnalyzerAgent - Deep sentiment analysis
+4. OutputValidatorAgent - Rule-based validation
+"""
+
+from .base_agent import BaseAgent
+from .content_preprocessor_agent import ContentPreprocessorAgent
+from .comment_preprocessor_agent import CommentPreprocessorAgent
+from .sabian_relevance_extraction_agent import SabianRelevanceExtractionAgent
+from .sabian_sentiment_analyzer_agent import SabianSentimentAnalyzerAgent
+from .output_validator_agent import OutputValidatorAgent
+
+# Legacy imports for backward compatibility
+from .preprocessor_agent import PreprocessorAgent
+from .relevance_validator_agent import RelevanceValidatorAgent
+from .sabian_analyzer_agent import SabianAnalyzerAgent
+
+__all__ = [
+    # Base
+    'BaseAgent',
+
+    # New agents (v4.0)
+    'ContentPreprocessorAgent',
+    'CommentPreprocessorAgent',
+    'SabianRelevanceExtractionAgent',
+    'SabianSentimentAnalyzerAgent',
+    'OutputValidatorAgent',
+
+    # Legacy agents (for backward compatibility)
+    'PreprocessorAgent',
+    'RelevanceValidatorAgent',
+    'SabianAnalyzerAgent'
+]

processing_brand_sentiment/workflow/agents/base_agent.py

@@ -0,0 +1,169 @@
+"""
+Base Agent class for all agents in the brand sentiment analysis workflow.
+Provides a common interface and structure for extensibility.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Dict, Any, Optional
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAgent(ABC):
+    """
+    Abstract base class for all agents in the brand sentiment analysis workflow.
+    Provides common functionality and enforces consistent interface.
+    """
+
+    def __init__(self, name: str, config: Dict[str, Any]):
+        """
+        Initialize the base agent.
+
+        Args:
+            name: Name of the agent
+            config: Configuration dictionary for the agent
+        """
+        self.name = name
+        self.config = config
+        self.model = config.get("model", "gpt-5-nano")
+        self.temperature = config.get("temperature", 0.2)
+        self.max_retries = config.get("max_retries", 3)
+        logger.info(f"Initialized {self.name} with model {self.model}")
+
+    @abstractmethod
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process input data and return results.
+        This method must be implemented by all concrete agent classes.
+
+        Args:
+            input_data: Dictionary containing input data for processing
+
+        Returns:
+            Dictionary containing processing results
+        """
+        pass
+
+    @abstractmethod
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate input data before processing.
+
+        Args:
+            input_data: Dictionary containing input data
+
+        Returns:
+            True if input is valid, False otherwise
+        """
+        pass
+
+    def get_name(self) -> str:
+        """Get the agent name."""
+        return self.name
+
+    def get_config(self) -> Dict[str, Any]:
+        """Get the agent configuration."""
+        return self.config
+
+    def log_processing(self, message: str, level: str = "info"):
+        """
+        Log processing information.
+
+        Args:
+            message: Log message
+            level: Log level (info, warning, error, debug)
+        """
+        log_method = getattr(logger, level, logger.info)
+        log_method(f"[{self.name}] {message}")
+
+    def handle_error(self, error: Exception, context: str = "") -> Dict[str, Any]:
+        """
+        Handle errors consistently across all agents.
+
+        Args:
+            error: The exception that occurred
+            context: Additional context about the error
+
+        Returns:
+            Error dictionary with details
+        """
+        error_msg = f"Error in {self.name}"
+        if context:
+            error_msg += f" ({context})"
+        error_msg += f": {str(error)}"
+
+        logger.error(error_msg)
+
+        return {
+            "success": False,
+            "error": str(error),
+            "agent": self.name,
+            "context": context
+        }
+
+    def _parse_llm_json_response(self, response_content: str) -> Dict[str, Any]:
+        """
+        Parse LLM response that may contain JSON wrapped in markdown code blocks.
+
+        Args:
+            response_content: Raw response content from LLM
+
+        Returns:
+            Parsed JSON dictionary
+
+        Raises:
+            json.JSONDecodeError: If JSON cannot be parsed
+        """
+        content = response_content.strip()
+
+        # Check if response is wrapped in markdown code block
+        if content.startswith("```json"):
+            # Remove ```json prefix and ``` suffix
+            content = content[7:]  # Remove ```json
+            if content.endswith("```"):
+                content = content[:-3]  # Remove trailing ```
+            content = content.strip()
+        elif content.startswith("```"):
+            # Remove generic ``` code block
+            content = content[3:]
+            if content.endswith("```"):
+                content = content[:-3]
+            content = content.strip()
+
+        # Parse the cleaned JSON
+        return json.loads(content)
+
+    def _safe_get(self, data: Dict[str, Any], key: str, default: Any = None) -> Any:
+        """
+        Safely get a value from a dictionary with a default.
+
+        Args:
+            data: Dictionary to get value from
+            key: Key to look up
+            default: Default value if key not found
+
+        Returns:
+            Value from dictionary or default
+        """
+        return data.get(key, default)
+
+    def _ensure_list(self, value: Any) -> list:
+        """
+        Ensure a value is a list.
+
+        Args:
+            value: Value to convert
+
+        Returns:
+            List version of value
+        """
+        if value is None:
+            return []
+        if isinstance(value, list):
+            return value
+        if isinstance(value, str):
+            # Try to parse as comma-separated
+            return [v.strip() for v in value.split(",") if v.strip()]
+        return [value]

processing_brand_sentiment/workflow/agents/comment_preprocessor_agent.py

@@ -0,0 +1,211 @@
+"""
+Comment Preprocessor Agent for brand sentiment analysis on social media comments.
+
+Extends ContentPreprocessorAgent but handles plain text (no HTML parsing).
+Builds context from content title, content description, and parent comment text
+instead of thread title and first post.
+
+Reuses: keyword sets, product alias mapping, language detection, relevance screening.
+Overrides: process() method for plain text handling and comment-specific context building.
+"""
+
+from typing import Dict, Any, Optional
+import logging
+
+from .content_preprocessor_agent import ContentPreprocessorAgent
+
+logger = logging.getLogger(__name__)
+
+
+class CommentPreprocessorAgent(ContentPreprocessorAgent):
+    """
+    Agent that preprocesses social media comments for brand sentiment analysis.
+
+    Inherits keyword detection, product alias mapping, language detection,
+    and relevance screening from ContentPreprocessorAgent.
+
+    Key differences from forum preprocessor:
+    - No HTML parsing (comments are plain text)
+    - Context built from content title + description + parent comment
+    - Different input field names (comment_text vs post_content)
+    """
+
+    def __init__(self, config: Dict[str, Any], brand_config: Dict[str, Any]):
+        """
+        Initialize the Comment Preprocessor Agent.
+
+        Args:
+            config: Agent configuration
+            brand_config: Brand-specific configuration with keywords, products, and aliases
+        """
+        super().__init__(config, brand_config)
+        self.name = "CommentPreprocessorAgent"
+
+        logger.info(
+            f"CommentPreprocessorAgent initialized with {len(self.primary_keywords)} primary keywords, "
+            f"{len(self.product_aliases)} product aliases"
+        )
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate that input contains required fields for comment processing.
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        required_fields = ["comment_sk", "comment_text"]
+        return all(field in input_data for field in required_fields)
+
+    def _build_comment_context(
+        self,
+        content_title: Optional[str] = None,
+        content_description: Optional[str] = None,
+        parent_comment_text: Optional[str] = None
+    ) -> str:
+        """
+        Build context string from social media content and parent comment information.
+
+        Args:
+            content_title: Title of the social media post/content
+            content_description: Description/message of the social media post
+            parent_comment_text: Text of the parent comment (if this is a reply)
+
+        Returns:
+            Formatted context string
+        """
+        context_parts = []
+
+        if content_title:
+            context_parts.append(f"Post title: {content_title}")
+
+        if content_description:
+            # Truncate if too long
+            truncated = content_description[:500] + "..." if len(content_description) > 500 else content_description
+            context_parts.append(f"Post description: {truncated}")
+
+        if parent_comment_text:
+            truncated = parent_comment_text[:500] + "..." if len(parent_comment_text) > 500 else parent_comment_text
+            context_parts.append(f"Parent comment: {truncated}")
+
+        return " | ".join(context_parts) if context_parts else ""
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a social media comment through the preprocessing pipeline.
+
+        Unlike forum posts, comments are plain text (no HTML parsing needed).
+        Context is built from content title, description, and parent comment.
+
+        Args:
+            input_data: Dictionary containing comment data with at least:
+                - comment_sk: Comment surrogate key
+                - comment_text: Raw comment text (plain text)
+                - content_title: Title of the post (optional)
+                - content_description: Description of the post (optional)
+                - parent_comment_text: Parent comment text if reply (optional)
+
+        Returns:
+            Dictionary with preprocessing results
+        """
+        try:
+            # Validate input
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields (comment_sk, comment_text)",
+                    **input_data
+                }
+
+            comment_text = input_data.get("comment_text", "")
+
+            # Step 1: Clean text (plain text - no HTML parsing needed)
+            cleaned_content = comment_text.strip() if comment_text else ""
+
+            # Check for empty content
+            if not cleaned_content or len(cleaned_content) < self.min_content_length:
+                return {
+                    "success": True,
+                    "cleaned_content": cleaned_content,
+                    "quoted_content": None,
+                    "is_empty": True,
+                    "preliminary_relevant": False,
+                    "needs_relevance_validation": False,
+                    **{k: v for k, v in input_data.items() if k != "comment_text"}
+                }
+
+            # Step 2: Check relevance (reused from parent class)
+            relevance_result = self._check_relevance(cleaned_content)
+            has_primary_keywords = relevance_result.get("has_primary_keywords", False)
+
+            # Step 3: Build comment context
+            raw_thread_context = self._build_comment_context(
+                content_title=input_data.get("content_title"),
+                content_description=input_data.get("content_description"),
+                parent_comment_text=input_data.get("parent_comment_text")
+            )
+
+            # Step 4: Detect language (reused from parent class)
+            lang_result = self._detect_language(cleaned_content, has_primary_keywords)
+
+            # Step 5: Extract product and competitor mentions (reused from parent class)
+            products_found = self._extract_mentioned_products(cleaned_content)
+            competitors_found = self._extract_mentioned_competitors(cleaned_content)
+
+            # Determine quoted content (parent comment serves as quoted context)
+            parent_comment = input_data.get("parent_comment_text")
+            has_parent = parent_comment is not None and str(parent_comment).strip() != ""
+
+            # Build result
+            result = {
+                "success": True,
+                "is_empty": False,
+
+                # Cleaned content
+                "cleaned_content": cleaned_content,
+                "quoted_content": parent_comment if has_parent else None,
+                "has_quote": has_parent,
+                "quoted_author": None,
+                "raw_thread_context": raw_thread_context,
+
+                # Language detection
+                "detected_language": lang_result["language"],
+                "language_code": lang_result["language_code"],
+                "is_english": lang_result["is_english"],
+                "language_confidence": lang_result["confidence"],
+                "language_detection_skipped": lang_result.get("detection_skipped", False),
+
+                # Relevance assessment
+                "preliminary_relevant": relevance_result["preliminary_relevant"],
+                "needs_relevance_validation": relevance_result["needs_relevance_validation"],
+                "relevance_keywords_found": relevance_result["found_keywords"],
+                "relevance_type": relevance_result["relevance_type"],
+                "relevance_confidence": relevance_result["relevance_confidence"],
+                "has_primary_keywords": has_primary_keywords,
+
+                # Initial extractions
+                "products_detected": products_found,
+                "competitors_detected": competitors_found,
+
+                # Preserve original data (exclude raw text to avoid duplication)
+                **{k: v for k, v in input_data.items() if k not in ["comment_text"]}
+            }
+
+            # Keep original content for reference
+            result["original_text"] = comment_text
+
+            self.log_processing(
+                f"Processed comment {input_data.get('comment_sk')}: "
+                f"lang={lang_result['language']}, "
+                f"relevant={relevance_result['preliminary_relevant']}, "
+                f"needs_validation={relevance_result['needs_relevance_validation']}, "
+                f"products={products_found}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, f"preprocessing comment {input_data.get('comment_sk')}")

processing_brand_sentiment/workflow/agents/content_preprocessor_agent.py

@@ -0,0 +1,570 @@
+"""
+Content Preprocessor Agent for brand sentiment analysis.
+Handles HTML parsing, text cleaning, language detection, product alias mapping,
+and initial relevance screening. This is a deterministic agent (no LLM calls).
+
+Enhanced version with:
+- Product alias mapping (B8 -> B8X)
+- Smart language detection (skip for short texts)
+- Always process if primary keywords found
+- Better content separation
+"""
+
+import re
+from typing import Dict, Any, List, Optional, Set
+from lingua import Language, LanguageDetectorBuilder
+import logging
+
+from .base_agent import BaseAgent
+from utils.html_parser import HTMLParser
+
+logger = logging.getLogger(__name__)
+
+
+class ContentPreprocessorAgent(BaseAgent):
+    """
+    Agent that preprocesses forum posts:
+    - Parses HTML to extract reply and quoted content
+    - Cleans and normalizes text
+    - Maps product aliases to canonical names
+    - Detects language (with smart handling for short texts)
+    - Performs initial keyword-based relevance screening
+    """
+
+    # Lingua to ISO 639-1 language code mapping
+    LINGUA_TO_ISO = {
+        Language.ENGLISH: "en",
+        Language.SPANISH: "es",
+        Language.FRENCH: "fr",
+        Language.GERMAN: "de",
+        Language.ITALIAN: "it",
+        Language.PORTUGUESE: "pt",
+        Language.RUSSIAN: "ru",
+        Language.JAPANESE: "ja",
+        Language.KOREAN: "ko",
+        Language.CHINESE: "zh",
+        Language.ARABIC: "ar",
+        Language.HINDI: "hi",
+        Language.DUTCH: "nl",
+        Language.SWEDISH: "sv",
+        Language.POLISH: "pl",
+        Language.TURKISH: "tr"
+    }
+
+    def __init__(self, config: Dict[str, Any], brand_config: Dict[str, Any]):
+        """
+        Initialize the Content Preprocessor Agent.
+
+        Args:
+            config: Agent configuration
+            brand_config: Brand-specific configuration with keywords, products, and aliases
+        """
+        super().__init__("ContentPreprocessorAgent", config)
+        self.brand_config = brand_config
+        self.html_parser = HTMLParser()
+
+        # Get preprocessing settings
+        preprocessing_config = brand_config.get("preprocessing", {})
+        self.min_length_for_lang_detection = preprocessing_config.get(
+            "min_length_for_language_detection", 50
+        )
+        self.default_language = preprocessing_config.get(
+            "default_language_for_short_text", "English"
+        )
+        self.always_process_primary = preprocessing_config.get(
+            "always_process_if_primary_keyword", True
+        )
+        self.min_content_length = preprocessing_config.get("min_content_length", 3)
+
+        # Initialize lingua detector
+        self.language_detector = LanguageDetectorBuilder.from_all_languages().build()
+
+        # Build keyword sets and alias mappings
+        self._build_keyword_sets()
+        self._build_alias_mappings()
+
+        logger.info(
+            f"ContentPreprocessorAgent initialized with {len(self.primary_keywords)} primary keywords, "
+            f"{len(self.product_aliases)} product aliases"
+        )
+
+    def _build_keyword_sets(self) -> None:
+        """Build keyword sets from brand configuration for efficient relevance checking."""
+        relevance_config = self.brand_config.get("relevance_keywords", {})
+
+        # Primary keywords - definitive Sabian mentions
+        primary = relevance_config.get("primary", {}).get("keywords", [])
+        self.primary_keywords: Set[str] = set(k.lower() for k in primary)
+
+        # Contextual keywords - need disambiguation (HH, AA)
+        contextual = relevance_config.get("contextual", {}).get("keywords", [])
+        self.contextual_keywords: Set[str] = set(k.lower() for k in contextual)
+
+        # Cymbal context keywords - help disambiguate contextual terms
+        cymbal_context = relevance_config.get("cymbal_context", {}).get("keywords", [])
+        self.cymbal_context_keywords: Set[str] = set(k.lower() for k in cymbal_context)
+
+        # Competitor names and aliases for detection
+        competitors = self.brand_config.get("brand", {}).get("competitors", [])
+        self.competitor_keywords: Set[str] = set()
+        self.competitor_name_map: Dict[str, str] = {}  # alias -> canonical name
+
+        for comp in competitors:
+            if isinstance(comp, dict):
+                name = comp.get("name", "")
+                self.competitor_keywords.add(name.lower())
+                self.competitor_name_map[name.lower()] = name
+                for alias in comp.get("aliases", []):
+                    alias_lower = alias.lower()
+                    self.competitor_keywords.add(alias_lower)
+                    self.competitor_name_map[alias_lower] = name
+            else:
+                comp_str = str(comp).lower()
+                self.competitor_keywords.add(comp_str)
+                self.competitor_name_map[comp_str] = str(comp)
+
+        # Product names
+        products = self.brand_config.get("brand", {}).get("products", [])
+        self.product_keywords: Set[str] = set(p.lower() for p in products)
+        self.products_list = products  # Keep original case
+
+        logger.debug(
+            f"Built keyword sets: {len(self.primary_keywords)} primary, "
+            f"{len(self.contextual_keywords)} contextual, "
+            f"{len(self.product_keywords)} products, "
+            f"{len(self.competitor_keywords)} competitor terms"
+        )
+
+    def _build_alias_mappings(self) -> None:
+        """Build product alias mappings from brand configuration."""
+        aliases = self.brand_config.get("brand", {}).get("product_aliases", {})
+
+        # Build alias -> canonical product mapping
+        self.product_aliases: Dict[str, str] = {}
+        for alias, canonical in aliases.items():
+            self.product_aliases[alias.lower()] = canonical
+
+        # Also add primary keywords that are aliases to contextual keywords
+        # e.g., "b8" should trigger contextual check since it maps to "B8X"
+        for alias in self.product_aliases.keys():
+            if alias not in self.primary_keywords:
+                self.contextual_keywords.add(alias)
+
+        logger.debug(f"Built {len(self.product_aliases)} product alias mappings")
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate that input contains required fields.
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        required_fields = ["post_id", "post_content"]
+        return all(field in input_data for field in required_fields)
+
+    def _detect_language(self, text: str, has_primary_keywords: bool = False) -> Dict[str, Any]:
+        """
+        Detect the language of text using lingua library.
+
+        Enhanced logic:
+        - Skip detection for short texts (< min_length_for_lang_detection chars)
+        - Always return English if primary Sabian keywords are found
+
+        Args:
+            text: Text to analyze
+            has_primary_keywords: Whether primary Sabian keywords were found
+
+        Returns:
+            Dictionary with language detection results
+        """
+        try:
+            cleaned_text = text.strip()
+
+            # If text is too short, default to English
+            if len(cleaned_text) < self.min_length_for_lang_detection:
+                return {
+                    "language": self.default_language,
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "low",
+                    "detection_skipped": True,
+                    "skip_reason": f"Text too short ({len(cleaned_text)} < {self.min_length_for_lang_detection} chars)"
+                }
+
+            # If primary keywords found and always_process_primary is True, treat as English
+            if has_primary_keywords and self.always_process_primary:
+                # Still try to detect, but override if non-English
+                detected = self.language_detector.detect_language_of(cleaned_text)
+
+                if detected == Language.ENGLISH:
+                    return {
+                        "language": "English",
+                        "language_code": "en",
+                        "is_english": True,
+                        "confidence": "high",
+                        "detection_skipped": False,
+                        "skip_reason": None
+                    }
+                else:
+                    # Primary keyword found but detected as non-English
+                    # Force to English since Sabian is explicitly mentioned
+                    lang_name = detected.name.capitalize() if detected else "Unknown"
+                    return {
+                        "language": "English",
+                        "language_code": "en",
+                        "is_english": True,
+                        "confidence": "medium",
+                        "detection_skipped": False,
+                        "skip_reason": None,
+                        "original_detected_language": lang_name,
+                        "override_reason": "Primary Sabian keyword found, treating as English"
+                    }
+
+            # Standard detection
+            detected = self.language_detector.detect_language_of(cleaned_text)
+
+            if detected is None:
+                return {
+                    "language": self.default_language,
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "low",
+                    "detection_skipped": False,
+                    "skip_reason": None
+                }
+
+            if detected == Language.ENGLISH:
+                return {
+                    "language": "English",
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "high",
+                    "detection_skipped": False,
+                    "skip_reason": None
+                }
+
+            lang_code = self.LINGUA_TO_ISO.get(detected, "unknown")
+            lang_name = detected.name.capitalize()
+
+            return {
+                "language": lang_name,
+                "language_code": lang_code,
+                "is_english": False,
+                "confidence": "high",
+                "detection_skipped": False,
+                "skip_reason": None
+            }
+
+        except Exception as e:
+            logger.warning(f"Language detection failed: {e}")
+            return {
+                "language": self.default_language,
+                "language_code": "en",
+                "is_english": True,
+                "confidence": "low",
+                "detection_skipped": False,
+                "skip_reason": None,
+                "detection_error": str(e)
+            }
+
+    def _normalize_product_mentions(self, found_products: List[str]) -> List[str]:
+        """
+        Normalize product mentions using alias mappings.
+
+        Args:
+            found_products: List of product terms found
+
+        Returns:
+            List of canonical product names
+        """
+        normalized = []
+        for product in found_products:
+            product_lower = product.lower()
+
+            # Check if it's an alias
+            if product_lower in self.product_aliases:
+                canonical = self.product_aliases[product_lower]
+                if canonical not in normalized:
+                    normalized.append(canonical)
+            # Check if it's a direct product match
+            elif product_lower in self.product_keywords:
+                # Find the original case version
+                for p in self.products_list:
+                    if p.lower() == product_lower:
+                        if p not in normalized:
+                            normalized.append(p)
+                        break
+
+        return normalized
+
+    def _check_relevance(self, text: str) -> Dict[str, Any]:
+        """
+        Check if text is relevant to the brand using keyword matching.
+
+        Enhanced to handle product aliases.
+
+        Returns:
+            Dictionary with relevance assessment
+        """
+        text_lower = text.lower()
+
+        # Tokenize for word boundary matching
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        # Also check for multi-word phrases (for aliases like "hand hammered")
+        all_aliases = set(self.product_aliases.keys())
+
+        # Check for primary keywords (definitive matches)
+        found_primary = self.primary_keywords.intersection(words)
+
+        # Check for product aliases in text
+        found_aliases = []
+        for alias in all_aliases:
+            if ' ' in alias:
+                # Multi-word alias - check in full text
+                if alias in text_lower:
+                    found_aliases.append(alias)
+            elif alias in words:
+                found_aliases.append(alias)
+
+        # Map aliases to canonical products
+        alias_products = []
+        for alias in found_aliases:
+            if alias in self.product_aliases:
+                canonical = self.product_aliases[alias]
+                if canonical not in alias_products:
+                    alias_products.append(canonical)
+
+        if found_primary or alias_products:
+            all_found = list(found_primary) + found_aliases
+            return {
+                "preliminary_relevant": True,
+                "needs_relevance_validation": False,
+                "found_keywords": all_found,
+                "mapped_products": alias_products,
+                "relevance_type": "primary",
+                "relevance_confidence": "high",
+                "has_primary_keywords": True
+            }
+
+        # Check for contextual keywords (need validation)
+        found_contextual = self.contextual_keywords.intersection(words)
+        if found_contextual:
+            # Check if there's cymbal context
+            found_cymbal_context = self.cymbal_context_keywords.intersection(words)
+            has_cymbal_context = len(found_cymbal_context) > 0
+
+            return {
+                "preliminary_relevant": True,
+                "needs_relevance_validation": True,
+                "found_keywords": list(found_contextual),
+                "cymbal_context_found": list(found_cymbal_context) if found_cymbal_context else [],
+                "has_cymbal_context": has_cymbal_context,
+                "mapped_products": [],
+                "relevance_type": "contextual",
+                "relevance_confidence": "medium" if has_cymbal_context else "low",
+                "has_primary_keywords": False
+            }
+
+        # Check for competitor mentions (might be comparative discussion)
+        found_competitors = self.competitor_keywords.intersection(words)
+        if found_competitors:
+            return {
+                "preliminary_relevant": False,
+                "needs_relevance_validation": True,
+                "found_keywords": list(found_competitors),
+                "mapped_products": [],
+                "relevance_type": "competitor_only",
+                "relevance_confidence": "low",
+                "has_primary_keywords": False
+            }
+
+        # No relevant keywords found
+        return {
+            "preliminary_relevant": False,
+            "needs_relevance_validation": False,
+            "found_keywords": [],
+            "mapped_products": [],
+            "relevance_type": "none",
+            "relevance_confidence": "high",
+            "has_primary_keywords": False
+        }
+
+    def _extract_mentioned_products(self, text: str) -> List[str]:
+        """
+        Extract product names mentioned in the text, including aliases.
+
+        Args:
+            text: Text to search
+
+        Returns:
+            List of canonical product names found
+        """
+        text_lower = text.lower()
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        found_products = []
+
+        # Check direct product mentions
+        for product in self.products_list:
+            if product.lower() in words:
+                if product not in found_products:
+                    found_products.append(product)
+
+        # Check aliases
+        for alias, canonical in self.product_aliases.items():
+            if ' ' in alias:
+                # Multi-word alias
+                if alias in text_lower:
+                    if canonical not in found_products:
+                        found_products.append(canonical)
+            elif alias in words:
+                if canonical not in found_products:
+                    found_products.append(canonical)
+
+        return found_products
+
+    def _extract_mentioned_competitors(self, text: str) -> List[str]:
+        """
+        Extract competitor brand names mentioned in the text.
+
+        Args:
+            text: Text to search
+
+        Returns:
+            List of canonical competitor names found
+        """
+        text_lower = text.lower()
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        found_competitors = set()
+
+        for alias in self.competitor_keywords:
+            if ' ' in alias:
+                # Multi-word check
+                if alias in text_lower:
+                    canonical = self.competitor_name_map.get(alias, alias)
+                    found_competitors.add(canonical)
+            elif alias in words:
+                canonical = self.competitor_name_map.get(alias, alias)
+                found_competitors.add(canonical)
+
+        return list(found_competitors)
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a forum post through preprocessing pipeline.
+
+        Args:
+            input_data: Dictionary containing post data with at least:
+                - post_id: Post identifier
+                - post_content: Raw HTML content
+                - thread_title: Thread title (optional)
+                - thread_first_post: First post content (optional)
+                - category_title: Category title (optional)
+                - category_topic: Category topic (optional)
+
+        Returns:
+            Dictionary with preprocessing results
+        """
+        try:
+            # Validate input
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    **input_data
+                }
+
+            post_content = input_data.get("post_content", "")
+
+            # Step 1: Parse HTML content
+            parsed = self.html_parser.parse_post_content(post_content)
+            reply_content = parsed.get("reply_content", "")
+            quoted_content = parsed.get("quoted_content")
+
+            # Check for empty content
+            if not reply_content or len(reply_content.strip()) < self.min_content_length:
+                return {
+                    "success": True,
+                    "cleaned_content": reply_content,
+                    "quoted_content": quoted_content,
+                    "is_empty": True,
+                    "preliminary_relevant": False,
+                    "needs_relevance_validation": False,
+                    **{k: v for k, v in input_data.items() if k != "post_content"}
+                }
+
+            # Step 2: Check relevance FIRST (needed for language detection logic)
+            relevance_result = self._check_relevance(reply_content)
+            has_primary_keywords = relevance_result.get("has_primary_keywords", False)
+
+            # Step 3: Build thread context (raw - will be summarized by extraction agent)
+            raw_thread_context = self.html_parser.build_thread_context(
+                thread_title=input_data.get("thread_title"),
+                first_post_content=input_data.get("thread_first_post"),
+                category_title=input_data.get("category_title"),
+                category_topic=input_data.get("category_topic")
+            )
+
+            # Step 4: Detect language (with smart handling)
+            lang_result = self._detect_language(reply_content, has_primary_keywords)
+
+            # Step 5: Extract product and competitor mentions from actual post content
+            products_found = self._extract_mentioned_products(reply_content)
+            competitors_found = self._extract_mentioned_competitors(reply_content)
+
+            # Build result
+            result = {
+                "success": True,
+                "is_empty": False,
+
+                # Cleaned content
+                "cleaned_content": reply_content,
+                "quoted_content": quoted_content,
+                "has_quote": parsed.get("has_quote", False),
+                "quoted_author": parsed.get("quoted_author"),
+                "raw_thread_context": raw_thread_context,
+
+                # Language detection
+                "detected_language": lang_result["language"],
+                "language_code": lang_result["language_code"],
+                "is_english": lang_result["is_english"],
+                "language_confidence": lang_result["confidence"],
+                "language_detection_skipped": lang_result.get("detection_skipped", False),
+
+                # Relevance assessment
+                "preliminary_relevant": relevance_result["preliminary_relevant"],
+                "needs_relevance_validation": relevance_result["needs_relevance_validation"],
+                "relevance_keywords_found": relevance_result["found_keywords"],
+                "relevance_type": relevance_result["relevance_type"],
+                "relevance_confidence": relevance_result["relevance_confidence"],
+                "has_primary_keywords": has_primary_keywords,
+
+                # Initial extractions
+                "products_detected": products_found,
+                "competitors_detected": competitors_found,
+
+                # Preserve original data
+                **{k: v for k, v in input_data.items() if k not in ["post_content"]}
+            }
+
+            # Keep original content for reference
+            result["original_content"] = post_content
+
+            self.log_processing(
+                f"Processed post {input_data.get('post_id')}: "
+                f"lang={lang_result['language']}, "
+                f"relevant={relevance_result['preliminary_relevant']}, "
+                f"needs_validation={relevance_result['needs_relevance_validation']}, "
+                f"products={products_found}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, f"preprocessing post {input_data.get('post_id')}")

processing_brand_sentiment/workflow/agents/output_validator_agent.py

@@ -0,0 +1,408 @@
+"""
+Output Validator Agent for brand sentiment analysis.
+
+This agent performs rule-based validation on the final output to ensure:
+1. All values are from predefined lists
+2. Logical consistency between fields
+3. Anomaly detection for manual review flagging
+
+This is a deterministic agent (no LLM calls) that acts as a quality gate.
+"""
+
+from typing import Dict, Any, List, Set
+import logging
+
+from .base_agent import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class OutputValidatorAgent(BaseAgent):
+    """
+    Agent that validates the final output for consistency and quality.
+
+    Performs rule-based checks without LLM calls to ensure data quality
+    and flag posts that may need manual review.
+    """
+
+    def __init__(
+        self,
+        config: Dict[str, Any],
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any]
+    ):
+        """
+        Initialize the Output Validator Agent.
+
+        Args:
+            config: Agent configuration
+            brand_config: Brand-specific configuration
+            analysis_categories: Category definitions for validation
+        """
+        super().__init__("OutputValidatorAgent", config)
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+
+        # Build valid value sets for validation
+        self._build_valid_value_sets()
+
+        logger.info("OutputValidatorAgent initialized")
+
+    def _build_valid_value_sets(self) -> None:
+        """Build sets of valid values for efficient validation."""
+        brand = self.brand_config.get("brand", {})
+
+        # Products
+        self.valid_products: Set[str] = set(
+            p.lower() for p in brand.get("products", [])
+        )
+        self.products_canonical = {p.lower(): p for p in brand.get("products", [])}
+
+        # Competitors
+        self.valid_competitors: Set[str] = set()
+        self.competitors_canonical = {}
+        for comp in brand.get("competitors", []):
+            if isinstance(comp, dict):
+                name = comp.get("name", "")
+                self.valid_competitors.add(name.lower())
+                self.competitors_canonical[name.lower()] = name
+
+        # Extract all category values
+        self.valid_values = {}
+
+        category_configs = {
+            "author_role": self.analysis_categories.get("author_role", {}),
+            "sabian_mention_context": self.analysis_categories.get("sabian_mention_context", {}),
+            "sentiment_level": self.analysis_categories.get("sentiment", {}),
+            "emotion_type": self.analysis_categories.get("emotions", {}),
+            "intents": self.analysis_categories.get("intents", {}),
+            "purchase_stage": self.analysis_categories.get("purchase_stage", {}),
+            "comparison_type": self.analysis_categories.get("comparison_type", {}),
+            "feedback_aspects": self.analysis_categories.get("feedback_aspects", {}),
+            "decision_drivers": self.analysis_categories.get("decision_drivers", {}),
+            "product_attributes": self.analysis_categories.get("product_attributes", {}),
+        }
+
+        for key, config in category_configs.items():
+            if "categories" in config:
+                self.valid_values[key] = set(
+                    c["value"].lower() for c in config["categories"]
+                )
+            elif "levels" in config:
+                self.valid_values[key] = set(
+                    c["value"].lower() for c in config["levels"]
+                )
+            else:
+                self.valid_values[key] = set()
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """Validate that input contains required fields."""
+        # The validator accepts any input - it will validate what's there
+        return True
+
+    def _validate_list_values(
+        self,
+        values: List[Any],
+        valid_set: Set[str],
+        field_name: str
+    ) -> Dict[str, Any]:
+        """
+        Validate list values against a set of valid values.
+
+        Returns:
+            Dictionary with validation results
+        """
+        if not values:
+            return {"valid": True, "invalid_values": [], "field": field_name}
+
+        invalid = []
+        for v in values:
+            if isinstance(v, str) and v.lower() not in valid_set:
+                invalid.append(v)
+
+        return {
+            "valid": len(invalid) == 0,
+            "invalid_values": invalid,
+            "field": field_name
+        }
+
+    def _validate_single_value(
+        self,
+        value: Any,
+        valid_set: Set[str],
+        field_name: str,
+        allow_none: bool = True
+    ) -> Dict[str, Any]:
+        """
+        Validate a single value against a set of valid values.
+
+        Returns:
+            Dictionary with validation results
+        """
+        if value is None:
+            return {"valid": allow_none, "invalid_value": None if allow_none else value, "field": field_name}
+
+        if isinstance(value, str) and value.lower() in valid_set:
+            return {"valid": True, "invalid_value": None, "field": field_name}
+
+        return {"valid": False, "invalid_value": value, "field": field_name}
+
+    def _check_logical_consistency(self, data: Dict[str, Any]) -> List[str]:
+        """
+        Check for logical consistency between fields.
+
+        Note: Empty products_mentioned is OK even when relevant - users may
+        discuss the Sabian brand generally without specific products.
+
+        Returns:
+            List of inconsistency warnings
+        """
+        warnings = []
+        is_relevant = data.get("is_relevant", False)
+
+        # Check 1: If not relevant, certain fields should be empty/null
+        if not is_relevant:
+            if data.get("sabian_mention_context"):
+                warnings.append(
+                    "sabian_mention_context should be null when is_relevant=False"
+                )
+            if data.get("sentiment_level") and data.get("sentiment_level") != "neutral":
+                warnings.append(
+                    "sentiment_level should be null/neutral when is_relevant=False"
+                )
+
+        # Check 2: Comparison type should only be set if comparing intent exists
+        if data.get("comparison_type"):
+            intents = data.get("intents", [])
+            if "comparing" not in intents:
+                warnings.append(
+                    "comparison_type is set but 'comparing' not in intents"
+                )
+
+        # Check 3: Author perspective fields consistency
+        # If author is giving advice (providing_information) without sharing experience,
+        # pain_points and delight_factors should typically be empty
+        intents = data.get("intents", [])
+        if "providing_information" in intents and "sharing_experience" not in intents:
+            if data.get("pain_points") or data.get("delight_factors"):
+                warnings.append(
+                    "pain_points/delight_factors set for advice-giving post without sharing_experience intent"
+                )
+
+        return warnings
+
+    def _fix_overlapping_feedback(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Fix overlapping values between pain_points and delight_factors.
+
+        Rule: The same aspect cannot be both a pain point and a delight factor.
+        Resolution: Use sentiment to determine which to keep, or clear both if neutral.
+
+        Args:
+            data: Dictionary with analysis results
+
+        Returns:
+            Updated dictionary with fixed pain_points and delight_factors
+        """
+        pain_points = data.get("pain_points", []) or []
+        delight_factors = data.get("delight_factors", []) or []
+
+        if not pain_points or not delight_factors:
+            return data
+
+        # Find overlapping values
+        pain_set = set(p.lower() if isinstance(p, str) else p for p in pain_points)
+        delight_set = set(d.lower() if isinstance(d, str) else d for d in delight_factors)
+        overlap = pain_set.intersection(delight_set)
+
+        if not overlap:
+            return data
+
+        # Get sentiment to determine which to keep
+        sentiment = data.get("sentiment_level", "neutral")
+
+        # Create new lists without overlapping values
+        if sentiment in ["positive", "very_positive"]:
+            # Keep in delight_factors, remove from pain_points
+            new_pain_points = [p for p in pain_points if p.lower() not in overlap]
+            new_delight_factors = delight_factors
+        elif sentiment in ["negative", "very_negative"]:
+            # Keep in pain_points, remove from delight_factors
+            new_pain_points = pain_points
+            new_delight_factors = [d for d in delight_factors if d.lower() not in overlap]
+        else:
+            # Neutral sentiment - clear both (can't determine intent)
+            new_pain_points = [p for p in pain_points if p.lower() not in overlap]
+            new_delight_factors = [d for d in delight_factors if d.lower() not in overlap]
+
+        # Update data
+        data["pain_points"] = new_pain_points
+        data["delight_factors"] = new_delight_factors
+
+        logger.debug(
+            f"Fixed overlapping feedback: removed {overlap} from "
+            f"{'pain_points' if sentiment in ['positive', 'very_positive'] else 'delight_factors' if sentiment in ['negative', 'very_negative'] else 'both'}"
+        )
+
+        return data
+
+    def _detect_anomalies(self, data: Dict[str, Any]) -> List[str]:
+        """
+        Detect anomalies that might need manual review.
+
+        Returns:
+            List of anomaly flags
+        """
+        anomalies = []
+
+        # Anomaly 1: Low confidence relevance
+        if data.get("is_relevant") and data.get("relevance_confidence") == "low":
+            anomalies.append("low_confidence_relevant")
+
+        # Anomaly 2: Sarcasm detected - sentiment might be inverted
+        if data.get("sarcasm_detected"):
+            anomalies.append("sarcasm_detected")
+
+        # Anomaly 3: Very short content marked as relevant
+        content = data.get("cleaned_content", "")
+        if data.get("is_relevant") and len(content) < 20:
+            anomalies.append("short_relevant_content")
+
+        # Anomaly 4: Switching behavior detected
+        comparison_type = data.get("comparison_type", "")
+        if comparison_type in ["switching_to_sabian", "switching_from_sabian"]:
+            anomalies.append(f"brand_switching_{comparison_type}")
+
+        return anomalies
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process and validate the analysis output.
+
+        Args:
+            input_data: Dictionary with all analysis results
+
+        Returns:
+            Dictionary with validation results added
+        """
+        try:
+            validation_errors = []
+            validation_warnings = []
+
+            # Skip detailed validation for non-relevant or skipped posts
+            if not input_data.get("is_relevant", False) or input_data.get("analysis_skipped", False):
+                return {
+                    **input_data,
+                    "validation_passed": True,
+                    "validation_errors": [],
+                    "validation_warnings": [],
+                    "validation_flags": [],
+                    "processing_status": "completed"
+                }
+
+            # Fix overlapping pain_points and delight_factors (safety net)
+            input_data = self._fix_overlapping_feedback(input_data)
+
+            # Validate products_mentioned
+            products_result = self._validate_list_values(
+                input_data.get("products_mentioned", []),
+                self.valid_products,
+                "products_mentioned"
+            )
+            if not products_result["valid"]:
+                validation_errors.append(
+                    f"Invalid products: {products_result['invalid_values']}"
+                )
+
+            # Validate competitors_mentioned
+            competitors_result = self._validate_list_values(
+                input_data.get("competitors_mentioned", []),
+                self.valid_competitors,
+                "competitors_mentioned"
+            )
+            if not competitors_result["valid"]:
+                validation_errors.append(
+                    f"Invalid competitors: {competitors_result['invalid_values']}"
+                )
+
+            # Validate categorical fields
+            categorical_validations = [
+                ("author_role", "author_role", True),
+                ("sabian_mention_context", "sabian_mention_context", True),
+                ("sentiment_level", "sentiment_level", True),
+                ("emotion_type", "emotion_type", True),
+                ("purchase_stage", "purchase_stage", True),
+                ("comparison_type", "comparison_type", True),
+            ]
+
+            for field, valid_key, allow_none in categorical_validations:
+                result = self._validate_single_value(
+                    input_data.get(field),
+                    self.valid_values.get(valid_key, set()),
+                    field,
+                    allow_none
+                )
+                if not result["valid"]:
+                    validation_errors.append(
+                        f"Invalid {field}: {result['invalid_value']}"
+                    )
+
+            # Validate list fields
+            list_validations = [
+                ("intents", "intents"),
+                ("product_attributes", "product_attributes"),
+                ("pain_points", "feedback_aspects"),
+                ("delight_factors", "feedback_aspects"),
+                ("decision_drivers", "decision_drivers"),
+            ]
+
+            for field, valid_key in list_validations:
+                result = self._validate_list_values(
+                    input_data.get(field, []),
+                    self.valid_values.get(valid_key, set()),
+                    field
+                )
+                if not result["valid"]:
+                    validation_warnings.append(
+                        f"Invalid values in {field}: {result['invalid_values']}"
+                    )
+
+            # Check logical consistency
+            consistency_warnings = self._check_logical_consistency(input_data)
+            validation_warnings.extend(consistency_warnings)
+
+            # Detect anomalies
+            anomalies = self._detect_anomalies(input_data)
+
+            # Determine overall validation status
+            validation_passed = len(validation_errors) == 0
+
+            # Set processing status
+            if validation_errors:
+                processing_status = "validation_failed"
+            elif anomalies:
+                processing_status = "completed_with_flags"
+            else:
+                processing_status = "completed"
+
+            result = {
+                **input_data,
+                "validation_passed": validation_passed,
+                "validation_errors": validation_errors,
+                "validation_warnings": validation_warnings,
+                "validation_flags": anomalies,
+                "processing_status": processing_status
+            }
+
+            if validation_errors or validation_warnings or anomalies:
+                self.log_processing(
+                    f"Validation complete: passed={validation_passed}, "
+                    f"errors={len(validation_errors)}, warnings={len(validation_warnings)}, "
+                    f"flags={anomalies}",
+                    "debug"
+                )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "output validation")

processing_brand_sentiment/workflow/agents/preprocessor_agent.py

@@ -0,0 +1,408 @@
+"""
+Preprocessor Agent for brand sentiment analysis.
+Handles HTML parsing, text cleaning, language detection, and initial relevance screening.
+This is a deterministic agent (no LLM calls except for language detection fallback).
+"""
+
+import re
+from typing import Dict, Any, List, Optional, Set
+from lingua import Language, LanguageDetectorBuilder
+import logging
+
+from .base_agent import BaseAgent
+from utils.html_parser import HTMLParser
+
+logger = logging.getLogger(__name__)
+
+
+class PreprocessorAgent(BaseAgent):
+    """
+    Agent that preprocesses forum posts:
+    - Parses HTML to extract reply and quoted content
+    - Cleans and normalizes text
+    - Detects language
+    - Performs initial keyword-based relevance screening
+    """
+
+    # Lingua to ISO 639-1 language code mapping
+    LINGUA_TO_ISO = {
+        Language.ENGLISH: "en",
+        Language.SPANISH: "es",
+        Language.FRENCH: "fr",
+        Language.GERMAN: "de",
+        Language.ITALIAN: "it",
+        Language.PORTUGUESE: "pt",
+        Language.RUSSIAN: "ru",
+        Language.JAPANESE: "ja",
+        Language.KOREAN: "ko",
+        Language.CHINESE: "zh",
+        Language.ARABIC: "ar",
+        Language.HINDI: "hi",
+        Language.DUTCH: "nl",
+        Language.SWEDISH: "sv",
+        Language.POLISH: "pl",
+        Language.TURKISH: "tr"
+    }
+
+    def __init__(self, config: Dict[str, Any], brand_config: Dict[str, Any]):
+        """
+        Initialize the Preprocessor Agent.
+
+        Args:
+            config: Agent configuration
+            brand_config: Brand-specific configuration with keywords and products
+        """
+        super().__init__("PreprocessorAgent", config)
+        self.brand_config = brand_config
+        self.html_parser = HTMLParser()
+
+        # Initialize lingua detector
+        self.language_detector = LanguageDetectorBuilder.from_all_languages().build()
+
+        # Build keyword sets for efficient lookup
+        self._build_keyword_sets()
+
+        logger.info("PreprocessorAgent initialized")
+
+    def _build_keyword_sets(self) -> None:
+        """Build keyword sets from brand configuration for efficient relevance checking."""
+        relevance_config = self.brand_config.get("relevance_keywords", {})
+
+        # Primary keywords - definitive Sabian mentions
+        primary = relevance_config.get("primary", {}).get("keywords", [])
+        self.primary_keywords: Set[str] = set(k.lower() for k in primary)
+
+        # Contextual keywords - need disambiguation (HH, AA)
+        contextual = relevance_config.get("contextual", {}).get("keywords", [])
+        self.contextual_keywords: Set[str] = set(k.lower() for k in contextual)
+
+        # Cymbal context keywords - help disambiguate contextual terms
+        cymbal_context = relevance_config.get("cymbal_context", {}).get("keywords", [])
+        self.cymbal_context_keywords: Set[str] = set(k.lower() for k in cymbal_context)
+
+        # Competitor names for detection
+        competitors = self.brand_config.get("brand", {}).get("competitors", [])
+        self.competitor_keywords: Set[str] = set()
+        for comp in competitors:
+            if isinstance(comp, dict):
+                self.competitor_keywords.add(comp.get("name", "").lower())
+                for alias in comp.get("aliases", []):
+                    self.competitor_keywords.add(alias.lower())
+            else:
+                self.competitor_keywords.add(str(comp).lower())
+
+        # Product names
+        products = self.brand_config.get("brand", {}).get("products", [])
+        self.product_keywords: Set[str] = set(p.lower() for p in products)
+
+        logger.info(f"Built keyword sets: {len(self.primary_keywords)} primary, "
+                   f"{len(self.contextual_keywords)} contextual, "
+                   f"{len(self.product_keywords)} products")
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate that input contains required fields.
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        required_fields = ["post_id", "post_content"]
+        return all(field in input_data for field in required_fields)
+
+    def _detect_language(self, text: str) -> Dict[str, Any]:
+        """
+        Detect the language of text using lingua library.
+
+        Args:
+            text: Text to analyze
+
+        Returns:
+            Dictionary with language detection results
+        """
+        try:
+            cleaned_text = text.strip()
+            if not cleaned_text or len(cleaned_text) < 3:
+                return {
+                    "language": "English",
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "low"
+                }
+
+            detected = self.language_detector.detect_language_of(cleaned_text)
+
+            if detected is None:
+                return {
+                    "language": "English",
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "low"
+                }
+
+            if detected == Language.ENGLISH:
+                return {
+                    "language": "English",
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "high"
+                }
+
+            lang_code = self.LINGUA_TO_ISO.get(detected, "unknown")
+            lang_name = detected.name.capitalize()
+
+            return {
+                "language": lang_name,
+                "language_code": lang_code,
+                "is_english": False,
+                "confidence": "high"
+            }
+
+        except Exception as e:
+            logger.warning(f"Language detection failed: {e}")
+            return {
+                "language": "English",
+                "language_code": "en",
+                "is_english": True,
+                "confidence": "low"
+            }
+
+    def _check_relevance(self, text: str) -> Dict[str, Any]:
+        """
+        Check if text is relevant to the brand using keyword matching.
+
+        Returns:
+            Dictionary with relevance assessment:
+            - preliminary_relevant: Initial relevance assessment
+            - needs_relevance_validation: True if contains ambiguous terms needing LLM check
+            - found_keywords: Keywords found in the text
+            - relevance_type: 'primary', 'contextual', or 'none'
+        """
+        text_lower = text.lower()
+
+        # Tokenize for word boundary matching
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        # Check for primary keywords (definitive matches)
+        found_primary = self.primary_keywords.intersection(words)
+        if found_primary:
+            return {
+                "preliminary_relevant": True,
+                "needs_relevance_validation": False,
+                "found_keywords": list(found_primary),
+                "relevance_type": "primary",
+                "relevance_confidence": "high"
+            }
+
+        # Check for contextual keywords (need validation)
+        found_contextual = self.contextual_keywords.intersection(words)
+        if found_contextual:
+            # Check if there's cymbal context
+            found_cymbal_context = self.cymbal_context_keywords.intersection(words)
+            has_cymbal_context = len(found_cymbal_context) > 0
+
+            return {
+                "preliminary_relevant": True,  # Potentially relevant
+                "needs_relevance_validation": True,  # Needs LLM confirmation
+                "found_keywords": list(found_contextual),
+                "cymbal_context_found": list(found_cymbal_context) if found_cymbal_context else [],
+                "has_cymbal_context": has_cymbal_context,
+                "relevance_type": "contextual",
+                "relevance_confidence": "medium" if has_cymbal_context else "low"
+            }
+
+        # Check for competitor mentions (might be comparative discussion)
+        found_competitors = self.competitor_keywords.intersection(words)
+        if found_competitors:
+            # Has competitor mention but no Sabian mention
+            # Could still be relevant in a comparison context
+            return {
+                "preliminary_relevant": False,
+                "needs_relevance_validation": True,  # LLM should check context
+                "found_keywords": list(found_competitors),
+                "relevance_type": "competitor_only",
+                "relevance_confidence": "low"
+            }
+
+        # No relevant keywords found
+        return {
+            "preliminary_relevant": False,
+            "needs_relevance_validation": False,
+            "found_keywords": [],
+            "relevance_type": "none",
+            "relevance_confidence": "high"
+        }
+
+    def _extract_mentioned_products(self, text: str) -> List[str]:
+        """
+        Extract product names mentioned in the text.
+
+        Args:
+            text: Text to search
+
+        Returns:
+            List of product names found
+        """
+        text_lower = text.lower()
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        found_products = []
+        products = self.brand_config.get("brand", {}).get("products", [])
+
+        for product in products:
+            if product.lower() in words:
+                found_products.append(product)
+
+        return found_products
+
+    def _extract_mentioned_competitors(self, text: str) -> List[str]:
+        """
+        Extract competitor names mentioned in the text.
+
+        Args:
+            text: Text to search
+
+        Returns:
+            List of competitor names found
+        """
+        text_lower = text.lower()
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        found_competitors = []
+        competitors = self.brand_config.get("brand", {}).get("competitors", [])
+
+        for comp in competitors:
+            if isinstance(comp, dict):
+                name = comp.get("name", "")
+                aliases = comp.get("aliases", [])
+
+                # Check name and aliases
+                if name.lower() in words:
+                    if name not in found_competitors:
+                        found_competitors.append(name)
+                else:
+                    for alias in aliases:
+                        if alias.lower() in words:
+                            if name not in found_competitors:
+                                found_competitors.append(name)
+                            break
+            else:
+                if str(comp).lower() in words:
+                    found_competitors.append(str(comp))
+
+        return found_competitors
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a forum post through preprocessing pipeline.
+
+        Args:
+            input_data: Dictionary containing post data with at least:
+                - post_id: Post identifier
+                - post_content: Raw HTML content
+                - thread_title: Thread title (optional)
+                - thread_first_post: First post content (optional)
+                - category_title: Category title (optional)
+                - category_topic: Category topic (optional)
+
+        Returns:
+            Dictionary with preprocessing results
+        """
+        try:
+            # Validate input
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    **input_data
+                }
+
+            post_content = input_data.get("post_content", "")
+
+            # Step 1: Parse HTML content
+            parsed = self.html_parser.parse_post_content(post_content)
+            reply_content = parsed.get("reply_content", "")
+            quoted_content = parsed.get("quoted_content")
+
+            # Check for empty content
+            if not reply_content or len(reply_content.strip()) < 3:
+                return {
+                    "success": True,
+                    "cleaned_content": reply_content,
+                    "quoted_content": quoted_content,
+                    "is_empty": True,
+                    "preliminary_relevant": False,
+                    "needs_relevance_validation": False,
+                    **{k: v for k, v in input_data.items() if k != "post_content"}
+                }
+
+            # Step 2: Build thread context
+            thread_context = self.html_parser.build_thread_context(
+                thread_title=input_data.get("thread_title"),
+                first_post_content=input_data.get("thread_first_post"),
+                category_title=input_data.get("category_title"),
+                category_topic=input_data.get("category_topic")
+            )
+
+            # Step 3: Detect language
+            lang_result = self._detect_language(reply_content)
+
+            # Step 4: Check relevance - ONLY on the actual post content, NOT quoted/context
+            # The quoted content and thread context are for understanding, not for relevance determination
+            relevance_result = self._check_relevance(reply_content)
+
+            # Step 5: Extract product and competitor mentions - ONLY from actual post content
+            # We don't want to extract from quoted content as that will be processed separately
+            products_found = self._extract_mentioned_products(reply_content)
+            competitors_found = self._extract_mentioned_competitors(reply_content)
+
+            # Build result
+            result = {
+                "success": True,
+                "is_empty": False,
+
+                # Cleaned content
+                "cleaned_content": reply_content,
+                "quoted_content": quoted_content,
+                "has_quote": parsed.get("has_quote", False),
+                "quoted_author": parsed.get("quoted_author"),
+                "thread_context": thread_context,
+
+                # Language detection
+                "detected_language": lang_result["language"],
+                "language_code": lang_result["language_code"],
+                "is_english": lang_result["is_english"],
+                "language_confidence": lang_result["confidence"],
+
+                # Relevance assessment
+                "preliminary_relevant": relevance_result["preliminary_relevant"],
+                "needs_relevance_validation": relevance_result["needs_relevance_validation"],
+                "relevance_keywords_found": relevance_result["found_keywords"],
+                "relevance_type": relevance_result["relevance_type"],
+                "relevance_confidence": relevance_result["relevance_confidence"],
+
+                # Initial extractions
+                "products_detected": products_found,
+                "competitors_detected": competitors_found,
+
+                # Preserve original data
+                **{k: v for k, v in input_data.items() if k not in ["post_content"]}
+            }
+
+            # Keep original content for reference
+            result["original_content"] = post_content
+
+            self.log_processing(
+                f"Processed post {input_data.get('post_id')}: "
+                f"lang={lang_result['language']}, "
+                f"relevant={relevance_result['preliminary_relevant']}, "
+                f"needs_validation={relevance_result['needs_relevance_validation']}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, f"preprocessing post {input_data.get('post_id')}")

processing_brand_sentiment/workflow/agents/relevance_validator_agent.py

@@ -0,0 +1,289 @@
+"""
+Relevance Validator Agent for brand sentiment analysis.
+Lightweight LLM-based agent that confirms whether ambiguous terms (HH, AA)
+refer to Sabian products or generic terms.
+"""
+
+from typing import Dict, Any
+import json
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+import logging
+
+from .base_agent import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class RelevanceValidatorAgent(BaseAgent):
+    """
+    Agent that validates whether posts with ambiguous terms (like HH, AA)
+    are actually referring to Sabian products or generic terms.
+
+    This is a lightweight LLM call specifically for disambiguation.
+    """
+
+    def __init__(self, config: Dict[str, Any], api_key: str, brand_config: Dict[str, Any]):
+        """
+        Initialize the Relevance Validator Agent.
+
+        Args:
+            config: Agent configuration
+            api_key: OpenAI API key
+            brand_config: Brand-specific configuration with product info
+        """
+        super().__init__("RelevanceValidatorAgent", config)
+        self.api_key = api_key
+        self.brand_config = brand_config
+
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key
+        )
+
+        # Build disambiguation context from brand config
+        self._build_disambiguation_context()
+
+        logger.info("RelevanceValidatorAgent initialized")
+
+    def _build_disambiguation_context(self) -> None:
+        """Build context strings for disambiguation from brand config."""
+        brand = self.brand_config.get("brand", {})
+        ambiguous = brand.get("ambiguous_terms", {})
+
+        self.disambiguation_info = {}
+        for term, info in ambiguous.items():
+            if isinstance(info, dict):
+                self.disambiguation_info[term] = {
+                    "description": info.get("description", ""),
+                    "context_clues": info.get("disambiguation_context", [])
+                }
+            else:
+                self.disambiguation_info[term] = {
+                    "description": str(info),
+                    "context_clues": []
+                }
+
+        # Product descriptions for context
+        self.product_descriptions = brand.get("product_descriptions", {})
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate that input contains required fields.
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        required = ["cleaned_content", "relevance_keywords_found"]
+        return all(field in input_data for field in required)
+
+    def _build_system_prompt(self) -> str:
+        """Build the system prompt for relevance validation."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+        products = self.brand_config.get("brand", {}).get("products", [])
+
+        # Build disambiguation rules
+        disambiguation_rules = []
+        for term, info in self.disambiguation_info.items():
+            desc = info.get("description", "")
+            clues = info.get("context_clues", [])
+            rule = f"- '{term}': {desc}"
+            if clues:
+                rule += f" Context clues for {brand_name}: {', '.join(clues)}"
+            disambiguation_rules.append(rule)
+
+        disambiguation_text = "\n".join(disambiguation_rules) if disambiguation_rules else "No specific disambiguation rules."
+
+        system_prompt = f"""You are an expert at identifying brand mentions in drum/cymbal forum discussions.
+
+Your task is to determine if the POST CONTENT itself discusses {brand_name} products.
+
+**CRITICAL RULE:**
+- You must determine relevance based ONLY on the POST CONTENT
+- The context (thread info, quoted/parent content) is provided to help you understand ambiguous terms
+- But if the POST CONTENT itself does not mention or discuss {brand_name}, it is NOT relevant
+- Example: If quoted content mentions Sabian but the post just says "Got it! Thanks!" → NOT relevant
+
+**{brand_name} Product Lines:**
+{', '.join(products)}
+
+**Ambiguous Terms to Watch For:**
+{disambiguation_text}
+
+**Key Disambiguation Rules:**
+- "HH" alone usually means "Hi-Hat" (a type of cymbal), NOT Sabian HH series
+- "HH" WITH Sabian context IN THE POST (e.g., "Sabian HH", "HH crashes", "my HH ride") likely refers to Sabian
+- "AA" alone might be a general abbreviation, NOT Sabian AA series
+- "AA" WITH Sabian context IN THE POST (e.g., "Sabian AA", "AA cymbals", "AA medium ride") likely refers to Sabian
+- Generic replies like "Thanks!", "Got it!", "Good point!" are NOT relevant even if context mentions {brand_name}
+
+**Return JSON with:**
+- is_relevant: boolean - true ONLY if the POST CONTENT itself discusses {brand_name} products
+- confidence: "high", "medium", or "low"
+- reason: brief explanation (1-2 sentences) - explain what IN THE POST made you decide
+- detected_products: list of {brand_name} products mentioned IN THE POST (empty if none)
+
+Return only valid JSON."""
+
+        return system_prompt
+
+    def validate_relevance(
+        self,
+        content: str,
+        keywords_found: list,
+        thread_context: str = "",
+        quoted_content: str = ""
+    ) -> Dict[str, Any]:
+        """
+        Validate whether content is relevant to the brand.
+
+        Args:
+            content: The cleaned post content
+            keywords_found: Keywords that triggered validation
+            thread_context: Thread context for additional context
+            quoted_content: Quoted content if any
+
+        Returns:
+            Dictionary with validation results
+        """
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+
+        # Build context for the LLM
+        context_parts = []
+        if thread_context:
+            context_parts.append(f"Thread context: {thread_context}")
+        if quoted_content:
+            context_parts.append(f"Replying to: {quoted_content[:300]}...")
+
+        context_str = "\n".join(context_parts) if context_parts else "No additional context."
+
+        user_prompt = f"""Determine if this POST CONTENT discusses {brand_name} cymbal products.
+
+**Keywords found in post:** {', '.join(keywords_found)}
+
+**CONTEXT (for understanding ambiguous terms only - do NOT base relevance on this):**
+{context_str}
+
+**POST CONTENT TO EVALUATE (base your relevance decision ONLY on this):**
+"{content}"
+
+Does the POST CONTENT itself discuss {brand_name} products? Remember: generic replies are NOT relevant even if context mentions {brand_name}. Return JSON only."""
+
+        try:
+            messages = [
+                SystemMessage(content=self._build_system_prompt()),
+                HumanMessage(content=user_prompt)
+            ]
+
+            response = self.llm.invoke(messages)
+            result = self._parse_llm_json_response(response.content)
+
+            return {
+                "success": True,
+                "is_relevant": result.get("is_relevant", False),
+                "relevance_confidence": result.get("confidence", "low"),
+                "relevance_reason": result.get("reason", ""),
+                "detected_products": result.get("detected_products", [])
+            }
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"JSON decode error in relevance validation: {e}", "warning")
+            # Default to relevant if we can't determine
+            return {
+                "success": True,
+                "is_relevant": True,
+                "relevance_confidence": "low",
+                "relevance_reason": "Could not parse LLM response, defaulting to relevant",
+                "detected_products": []
+            }
+
+        except Exception as e:
+            self.log_processing(f"Relevance validation error: {e}", "error")
+            return {
+                "success": False,
+                "is_relevant": True,  # Default to relevant on error
+                "relevance_confidence": "low",
+                "relevance_reason": f"Error during validation: {str(e)}",
+                "detected_products": [],
+                "error": str(e)
+            }
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a post to validate its relevance to the brand.
+
+        Args:
+            input_data: Dictionary containing:
+                - cleaned_content: Cleaned post text
+                - relevance_keywords_found: Keywords that triggered validation
+                - thread_context: Optional thread context
+                - quoted_content: Optional quoted content
+
+        Returns:
+            Dictionary with validation results and original data
+        """
+        try:
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    "is_relevant": True,  # Default to relevant
+                    "relevance_confidence": "low",
+                    **input_data
+                }
+
+            # Check if validation is actually needed
+            if not input_data.get("needs_relevance_validation", False):
+                # No validation needed, use preliminary assessment
+                return {
+                    "success": True,
+                    "is_relevant": input_data.get("preliminary_relevant", False),
+                    "relevance_confidence": input_data.get("relevance_confidence", "high"),
+                    "relevance_reason": "No validation needed - preliminary assessment used",
+                    "validation_performed": False,
+                    **input_data
+                }
+
+            # Perform LLM validation
+            validation_result = self.validate_relevance(
+                content=input_data.get("cleaned_content", ""),
+                keywords_found=input_data.get("relevance_keywords_found", []),
+                thread_context=input_data.get("thread_context", ""),
+                quoted_content=input_data.get("quoted_content", "")
+            )
+
+            # Merge results
+            result = {
+                **input_data,
+                "is_relevant": validation_result["is_relevant"],
+                "relevance_confidence": validation_result["relevance_confidence"],
+                "relevance_reason": validation_result["relevance_reason"],
+                "validation_performed": True,
+                "success": validation_result["success"]
+            }
+
+            # Update products detected if LLM found any
+            if validation_result.get("detected_products"):
+                existing_products = input_data.get("products_detected", [])
+                llm_products = validation_result["detected_products"]
+                # Merge without duplicates
+                all_products = list(set(existing_products + llm_products))
+                result["products_detected"] = all_products
+
+            if "error" in validation_result:
+                result["validation_error"] = validation_result["error"]
+
+            self.log_processing(
+                f"Validated relevance for post: is_relevant={result['is_relevant']}, "
+                f"confidence={result['relevance_confidence']}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "relevance validation")

processing_brand_sentiment/workflow/agents/sabian_analyzer_agent.py

@@ -0,0 +1,388 @@
+"""
+Sabian Analyzer Agent for comprehensive brand sentiment analysis.
+LLM-based agent that extracts products, competitors, sentiment, intents,
+pain points, and other brand intelligence from forum posts.
+"""
+
+from typing import Dict, Any, List
+import json
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+import logging
+
+from .base_agent import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class SabianAnalyzerAgent(BaseAgent):
+    """
+    Comprehensive brand analysis agent for Sabian cymbal discussions.
+    """
+
+    def __init__(
+        self,
+        config: Dict[str, Any],
+        api_key: str,
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any]
+    ):
+        super().__init__("SabianAnalyzerAgent", config)
+        self.api_key = api_key
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key
+        )
+
+        # Pre-compute valid values for validation
+        self._valid_values = self._compute_valid_values()
+        logger.info("SabianAnalyzerAgent initialized")
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        required = ["cleaned_content", "is_relevant"]
+        return all(field in input_data for field in required)
+
+    def _compute_valid_values(self) -> Dict[str, List[str]]:
+        """Pre-compute all valid values from config for validation."""
+        valid = {}
+
+        # Products from brand config
+        valid["products"] = self.brand_config.get("brand", {}).get("products", [])
+
+        # Competitors
+        competitor_names = []
+        for comp in self.brand_config.get("brand", {}).get("competitors", []):
+            if isinstance(comp, dict):
+                competitor_names.append(comp.get("name", ""))
+        valid["competitors"] = competitor_names
+
+        # Extract category values from analysis_categories
+        category_map = {
+            "author_role": "author_role",
+            "sabian_mention_context": "sabian_mention_context",
+            "sentiment_level": "sentiment",
+            "emotion_type": "emotions",
+            "intents": "intents",
+            "purchase_stage": "purchase_stage",
+            "comparison_type": "comparison_type",
+            "feedback_aspects": "feedback_aspects",
+            "decision_drivers": "decision_drivers",
+            "product_attributes": "product_attributes",
+        }
+
+        for key, config_key in category_map.items():
+            config_section = self.analysis_categories.get(config_key, {})
+            if "categories" in config_section:
+                valid[key] = [c["value"] for c in config_section["categories"]]
+            elif "levels" in config_section:
+                valid[key] = [c["value"] for c in config_section["levels"]]
+            else:
+                valid[key] = []
+
+        return valid
+
+    def _get_category_list(self, key: str) -> List[str]:
+        """Get list of valid values for a category."""
+        return self._valid_values.get(key, [])
+
+    def _build_system_prompt(self) -> str:
+        """Build optimized system prompt for brand analysis."""
+        brand = self.brand_config.get("brand", {})
+        brand_name = brand.get("name", "Sabian")
+        products = brand.get("products", [])
+
+        competitors = [c.get("name", "") for c in brand.get("competitors", []) if isinstance(c, dict)]
+
+        # Get all valid values
+        v = self._valid_values
+
+        return f"""You are a brand analyst extracting insights from forum posts about {brand_name} cymbals.
+
+## STRICT RULES
+1. Extract ONLY from POST CONTENT, never from quoted/context text
+2. Use ONLY values from the lists below - return null/[] if no match
+3. Sentiment must be about {brand_name} specifically, NOT overall post tone
+4. pain_points/delight_factors use SAME value list (feedback_aspects) - classification determines positive vs negative
+
+## VALID VALUES
+
+**{brand_name} Products:** {products}
+**Competitors:** {competitors}
+
+| Field | Valid Values |
+|-------|--------------|
+| author_role | {v.get('author_role', [])} |
+| sabian_mention_context | {v.get('sabian_mention_context', [])} |
+| sentiment_level | {v.get('sentiment_level', [])} |
+| emotion_type | {v.get('emotion_type', [])} |
+| intents (multi) | {v.get('intents', [])} |
+| purchase_stage | {v.get('purchase_stage', [])} |
+| comparison_type | {v.get('comparison_type', [])} |
+| feedback_aspects | {v.get('feedback_aspects', [])} |
+| decision_drivers | {v.get('decision_drivers', [])} |
+| product_attributes | {v.get('product_attributes', [])} |
+
+## KEY DISTINCTIONS
+
+**Sentiment vs Intent:**
+- sentiment_level = How author FEELS about {brand_name} (positive/negative/neutral)
+- praising/criticizing intent = Author is actively ENDORSING or WARNING others
+
+**Author-only fields (null if giving advice to others):**
+- purchase_stage, decision_drivers, pain_points, delight_factors
+
+**Example - Sabian-specific sentiment:**
+Post: "Love my new drum kit! The SBR cymbals sound terrible though."
+- Overall post: positive (happy about kit)
+- {brand_name} sentiment: NEGATIVE (dislikes SBR sound)
+- pain_points: ["sound_quality"]
+
+## OUTPUT JSON
+```json
+{{
+  "author_role": "value from list",
+  "sabian_mention_context": "value from list",
+  "sentiment_level": "value from list",
+  "emotion_type": "value or null",
+  "sentiment_confidence": "high|medium|low",
+  "sarcasm_detected": false,
+  "products_mentioned": [],
+  "product_attributes": [],
+  "competitors_mentioned": [],
+  "competitor_products_owned": [],
+  "comparison_type": "value or null",
+  "intents": [],
+  "purchase_stage": "value or null",
+  "decision_drivers": [],
+  "pain_points": [],
+  "delight_factors": [],
+  "analysis_notes": "1-2 sentences on key {brand_name}-specific insights"
+}}
+```
+
+Return ONLY valid JSON."""
+
+    def analyze_post(
+        self,
+        content: str,
+        thread_context: str = "",
+        quoted_content: str = ""
+    ) -> Dict[str, Any]:
+        """Perform brand analysis on a post."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+
+        context_str = ""
+        if thread_context:
+            context_str += f"[Thread: {thread_context[:200]}] "
+        if quoted_content:
+            context_str += f"[Replying to: {quoted_content[:200]}...]"
+
+        user_prompt = f"""Analyze this post about {brand_name}.
+
+CONTEXT (for understanding only, DO NOT extract from): {context_str or "None"}
+
+POST CONTENT (extract from THIS only):
+"{content}"
+
+Return JSON only."""
+
+        try:
+            messages = [
+                SystemMessage(content=self._build_system_prompt()),
+                HumanMessage(content=user_prompt)
+            ]
+
+            response = self.llm.invoke(messages)
+            result = self._parse_llm_json_response(response.content)
+            validated = self._validate_and_normalize(result)
+
+            return {"success": True, **validated}
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"JSON decode error: {e}", "warning")
+            return {
+                "success": False,
+                "error": f"JSON parse error: {str(e)}",
+                "sentiment_level": "neutral",
+                "intents": ["general_discussion"]
+            }
+        except Exception as e:
+            self.log_processing(f"Analysis error: {e}", "error")
+            return {"success": False, "error": str(e)}
+
+    def _validate_single(self, value: Any, valid_list: List[str], default: Any = None) -> Any:
+        """Validate single value against list, return canonical form or default."""
+        if value is None:
+            return default
+        if isinstance(value, str):
+            val_lower = value.lower()
+            for v in valid_list:
+                if v.lower() == val_lower:
+                    return v
+        return default
+
+    def _validate_list(self, values: Any, valid_list: List[str]) -> List[str]:
+        """Validate list values, return only valid items in canonical form."""
+        if not values:
+            return []
+        if not isinstance(values, list):
+            values = [values]
+
+        validated = []
+        valid_lower = {v.lower(): v for v in valid_list}
+        for val in values:
+            if isinstance(val, str) and val.lower() in valid_lower:
+                validated.append(valid_lower[val.lower()])
+        return validated
+
+    def _validate_and_normalize(self, result: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate all fields against predefined values and normalize."""
+        v = self._valid_values
+
+        normalized = {
+            # Classification
+            "author_role": self._validate_single(
+                result.get("author_role"), v["author_role"], "unknown"
+            ),
+            "sabian_mention_context": self._validate_single(
+                result.get("sabian_mention_context"), v["sabian_mention_context"], "casual_mention"
+            ),
+
+            # Sentiment
+            "sentiment_level": self._validate_single(
+                result.get("sentiment_level"), v["sentiment_level"], "neutral"
+            ),
+            "emotion_type": self._validate_single(
+                result.get("emotion_type"), v["emotion_type"], None
+            ),
+            "sentiment_confidence": result.get("sentiment_confidence", "medium"),
+            "sarcasm_detected": bool(result.get("sarcasm_detected", False)),
+
+            # Products
+            "products_mentioned": self._validate_list(
+                result.get("products_mentioned"), v["products"]
+            ),
+            "product_attributes": self._validate_list(
+                result.get("product_attributes"), v["product_attributes"]
+            ),
+
+            # Competitors
+            "competitors_mentioned": self._validate_list(
+                result.get("competitors_mentioned"), v["competitors"]
+            ),
+            "competitor_products_owned": self._validate_list(
+                result.get("competitor_products_owned"), v["competitors"]
+            ),
+            "comparison_type": self._validate_single(
+                result.get("comparison_type"), v["comparison_type"], None
+            ),
+
+            # Intents
+            "intents": self._validate_list(
+                result.get("intents"), v["intents"]
+            ) or ["general_discussion"],
+
+            # Author journey (null if advising others)
+            "purchase_stage": self._validate_single(
+                result.get("purchase_stage"), v["purchase_stage"], None
+            ),
+            "decision_drivers": self._validate_list(
+                result.get("decision_drivers"), v["decision_drivers"]
+            ),
+
+            # Feedback - both use feedback_aspects
+            "pain_points": self._validate_list(
+                result.get("pain_points"), v["feedback_aspects"]
+            ),
+            "delight_factors": self._validate_list(
+                result.get("delight_factors"), v["feedback_aspects"]
+            ),
+
+            # Notes
+            "analysis_notes": result.get("analysis_notes", ""),
+        }
+
+        # Log filtered values for debugging
+        for field in ["products_mentioned", "product_attributes", "pain_points", "delight_factors"]:
+            original = result.get(field, [])
+            if isinstance(original, list) and len(original) > len(normalized[field]):
+                filtered = set(str(x) for x in original) - set(normalized[field])
+                if filtered:
+                    logger.debug(f"Filtered invalid {field}: {filtered}")
+
+        return normalized
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Process a post through brand analysis."""
+        try:
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    **input_data
+                }
+
+            # Skip non-relevant posts
+            if not input_data.get("is_relevant", False):
+                return {
+                    "success": True,
+                    "analysis_skipped": True,
+                    "analysis_skip_reason": "Post marked as not relevant",
+                    "author_role": None,
+                    "sabian_mention_context": None,
+                    "sentiment_level": None,
+                    "emotion_type": None,
+                    "products_mentioned": [],
+                    "competitors_mentioned": [],
+                    "competitor_products_owned": [],
+                    "intents": [],
+                    "purchase_stage": None,
+                    "decision_drivers": [],
+                    "pain_points": [],
+                    "delight_factors": [],
+                    **input_data
+                }
+
+            # Skip non-English posts
+            if not input_data.get("is_english", True):
+                return {
+                    "success": True,
+                    "analysis_skipped": True,
+                    "analysis_skip_reason": f"Non-English: {input_data.get('detected_language')}",
+                    "author_role": None,
+                    "sabian_mention_context": None,
+                    "sentiment_level": None,
+                    "emotion_type": None,
+                    "intents": [],
+                    "competitor_products_owned": [],
+                    **input_data
+                }
+
+            # Perform analysis
+            analysis_result = self.analyze_post(
+                content=input_data.get("cleaned_content", ""),
+                thread_context=input_data.get("thread_context", ""),
+                quoted_content=input_data.get("quoted_content", "")
+            )
+
+            result = {
+                **input_data,
+                **analysis_result,
+                "analysis_skipped": False
+            }
+
+            self.log_processing(
+                f"Analyzed: sentiment={result.get('sentiment_level')}, "
+                f"products={len(result.get('products_mentioned', []))}, "
+                f"intents={result.get('intents', [])}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "brand analysis")

processing_brand_sentiment/workflow/agents/sabian_relevance_extraction_agent.py

@@ -0,0 +1,431 @@
+"""
+Sabian Relevance & Extraction Agent for brand sentiment analysis.
+
+This agent performs two critical functions:
+1. Determines relevance with HIGH confidence using strict rules
+2. Extracts verifiable facts (products, author role, context summary)
+
+Key Design Principles:
+- Strict product matching: ONLY return products from predefined list
+- Competitor awareness: Know what products belong to competitors
+- Conservative relevance: When uncertain, mark as NOT relevant
+- Thread context summarization: Provide clean, concise context for next agent
+"""
+
+from typing import Dict, Any, List
+import json
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+import logging
+
+from .base_agent import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class SabianRelevanceExtractionAgent(BaseAgent):
+    """
+    Agent that validates relevance and extracts key facts from posts.
+
+    This agent is the first LLM call in the pipeline and serves as the
+    gatekeeper for relevance while also extracting structured information
+    for downstream analysis.
+    """
+
+    def __init__(
+        self,
+        config: Dict[str, Any],
+        api_key: str,
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any]
+    ):
+        """
+        Initialize the Relevance & Extraction Agent.
+
+        Args:
+            config: Agent configuration
+            api_key: OpenAI API key
+            brand_config: Brand-specific configuration with products and competitors
+            analysis_categories: Category definitions for validation
+        """
+        super().__init__("SabianRelevanceExtractionAgent", config)
+        self.api_key = api_key
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key
+        )
+
+        # Pre-compute valid values
+        self._build_valid_values()
+        self._build_competitor_product_warnings()
+
+        logger.info("SabianRelevanceExtractionAgent initialized")
+
+    def _build_valid_values(self) -> None:
+        """Build valid value lists for validation."""
+        brand = self.brand_config.get("brand", {})
+
+        # Products
+        self.valid_products = brand.get("products", [])
+
+        # Competitors (brand names only)
+        self.valid_competitors = []
+        for comp in brand.get("competitors", []):
+            if isinstance(comp, dict):
+                self.valid_competitors.append(comp.get("name", ""))
+            else:
+                self.valid_competitors.append(str(comp))
+
+        # Author roles from categories
+        author_role_config = self.analysis_categories.get("author_role", {})
+        self.valid_author_roles = [
+            c["value"] for c in author_role_config.get("categories", [])
+        ]
+
+        # Sabian mention context from categories
+        mention_context_config = self.analysis_categories.get("sabian_mention_context", {})
+        self.valid_mention_contexts = [
+            c["value"] for c in mention_context_config.get("categories", [])
+        ]
+
+    def _build_competitor_product_warnings(self) -> None:
+        """Build list of competitor products to warn about in prompts."""
+        warnings = self.brand_config.get("brand", {}).get("competitor_products_warning", {})
+
+        self.competitor_products_by_brand = {}
+        for key, products in warnings.items():
+            if key == "description":
+                continue
+            # Extract brand name from key (e.g., "paiste_products" -> "Paiste")
+            brand_name = key.replace("_products", "").capitalize()
+            self.competitor_products_by_brand[brand_name] = products
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """Validate input contains required fields."""
+        required = ["cleaned_content"]
+        return all(field in input_data for field in required)
+
+    def _build_system_prompt(self) -> str:
+        """Build the system prompt for relevance and extraction."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+
+        # Build competitor product warnings
+        competitor_warnings = []
+        for brand, products in self.competitor_products_by_brand.items():
+            products_str = ", ".join(f'"{p}"' for p in products[:5])  # Limit to 5 examples
+            if len(products) > 5:
+                products_str += f" (and {len(products)-5} more)"
+            competitor_warnings.append(f"- {brand}: {products_str}")
+
+        competitor_warnings_text = "\n".join(competitor_warnings) if competitor_warnings else "None specified"
+
+        return f"""You are a brand mention extractor for {brand_name} cymbals. Your job is to:
+1. Determine if the POST CONTENT discusses {brand_name} products or brand
+2. Extract ONLY verifiable facts, not interpretations
+
+## CRITICAL RULES
+
+### Rule 1: Relevance Based on POST CONTENT Only
+- The post is relevant ONLY if the POST CONTENT itself mentions {brand_name} brand or products
+- Quoted/parent content mentioning {brand_name} does NOT make the post relevant
+- Generic replies ("Thanks!", "Got it!", "Good point!") are NEVER relevant
+- Posts can be relevant even without specific product mentions if they discuss the {brand_name} brand
+
+### Rule 2: Strict Product Matching
+{brand_name.upper()} PRODUCTS (use ONLY these exact values):
+{self.valid_products}
+
+CRITICAL:
+- Return ONLY products from this exact list above
+- If you see a product not in this list, do NOT include it
+- Return empty list [] if no products from the list are mentioned
+- It's OK to have empty products_mentioned if the post discusses {brand_name} brand generally
+
+### Rule 3: Competitor Product Awareness
+These products belong to COMPETITORS, NOT {brand_name}:
+{competitor_warnings_text}
+
+COMPETITOR BRANDS: {self.valid_competitors}
+- Only return competitor BRAND names in competitors_mentioned (not their products)
+- If you see "2002", "Signature", "Sound Edge", "Formula 602" - these are PAISTE, not {brand_name}
+- If you see "K Custom", "A Custom" - these are ZILDJIAN, not {brand_name}
+
+### Rule 4: Thread Context Summary
+- Summarize thread context in 1-2 sentences MAXIMUM
+- Focus only on what helps understand what the post is responding to
+- If thread is about unrelated topics (pizza, general life), say so briefly
+- Keep it factual and concise
+
+### Rule 5: Author Role Classification
+Determine the author's relationship to {brand_name}:
+- current_owner: Currently owns/uses {brand_name} products
+- past_owner: Previously owned but sold/replaced
+- potential_buyer: Considering purchasing {brand_name}
+- never_owned: Explicitly states they don't own {brand_name}
+- unknown: Cannot determine from post content
+
+### Rule 6: Mention Context Classification
+How prominently is {brand_name} discussed IN THE POST CONTENT:
+- primary_focus: {brand_name} is the main topic of the post
+- significant_mention: {brand_name} discussed with some detail, but not main focus
+- casual_mention: Brief mention among other topics
+- comparison_context: Mentioned while comparing to competitors
+- null: Not relevant (use when is_relevant=false)
+
+## OUTPUT FORMAT
+Return ONLY valid JSON with these exact fields:
+```json
+{{
+    "is_relevant": true/false,
+    "relevance_confidence": "high" | "medium" | "low",
+    "relevance_reason": "1-2 sentences explaining your decision",
+    "products_mentioned": [],
+    "sabian_mention_context": "value from list" | null,
+    "author_role": "value from list",
+    "competitors_mentioned": [],
+    "thread_context_summary": "1-2 sentence summary of thread context"
+}}
+```
+
+IMPORTANT: Return ONLY the JSON object, no additional text."""
+
+    def _build_user_prompt(
+        self,
+        content: str,
+        quoted_content: str,
+        raw_thread_context: str,
+        keywords_found: List[str]
+    ) -> str:
+        """Build the user prompt with post content and context."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+
+        context_section = ""
+        if raw_thread_context:
+            # Truncate if too long
+            truncated_context = raw_thread_context[:1000] if len(raw_thread_context) > 1000 else raw_thread_context
+            context_section += f"THREAD CONTEXT (for understanding only):\n{truncated_context}\n\n"
+
+        if quoted_content:
+            truncated_quote = quoted_content[:500] if len(quoted_content) > 500 else quoted_content
+            context_section += f"QUOTED/PARENT CONTENT (for understanding only):\n{truncated_quote}\n\n"
+
+        keywords_info = ""
+        if keywords_found:
+            keywords_info = f"Keywords detected by preprocessor: {', '.join(keywords_found)}\n\n"
+
+        return f"""Analyze this post for {brand_name} relevance and extract facts.
+
+{keywords_info}{context_section}POST CONTENT TO EVALUATE (base your decision ONLY on this):
+\"\"\"{content}\"\"\"
+
+Remember:
+- is_relevant=true ONLY if POST CONTENT discusses {brand_name}
+- products_mentioned must be from the exact product list provided
+- competitors_mentioned should be brand names only (Zildjian, Paiste, etc.)
+- thread_context_summary should be 1-2 sentences max
+
+Return JSON only."""
+
+    def extract_and_validate(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Perform relevance check and fact extraction.
+
+        Args:
+            input_data: Preprocessed post data
+
+        Returns:
+            Dictionary with extraction results
+        """
+        content = input_data.get("cleaned_content", "")
+        quoted_content = input_data.get("quoted_content", "")
+        raw_thread_context = input_data.get("raw_thread_context", "")
+        keywords_found = input_data.get("relevance_keywords_found", [])
+
+        try:
+            messages = [
+                SystemMessage(content=self._build_system_prompt()),
+                HumanMessage(content=self._build_user_prompt(
+                    content, quoted_content, raw_thread_context, keywords_found
+                ))
+            ]
+
+            response = self.llm.invoke(messages)
+            result = self._parse_llm_json_response(response.content)
+
+            # Validate and normalize the response
+            validated = self._validate_response(result)
+
+            return {
+                "success": True,
+                **validated
+            }
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"JSON decode error: {e}", "warning")
+            return {
+                "success": False,
+                "error": f"JSON parse error: {str(e)}",
+                "is_relevant": False,
+                "relevance_confidence": "low",
+                "relevance_reason": "Failed to parse LLM response"
+            }
+
+        except Exception as e:
+            self.log_processing(f"Extraction error: {e}", "error")
+            return {
+                "success": False,
+                "error": str(e),
+                "is_relevant": False,
+                "relevance_confidence": "low",
+                "relevance_reason": f"Error during extraction: {str(e)}"
+            }
+
+    def _validate_response(self, result: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate and normalize LLM response against allowed values."""
+
+        # Validate products
+        products = result.get("products_mentioned", [])
+        if not isinstance(products, list):
+            products = []
+        valid_products = [
+            p for p in products
+            if any(p.lower() == vp.lower() for vp in self.valid_products)
+        ]
+        # Normalize to canonical case
+        normalized_products = []
+        for p in valid_products:
+            for vp in self.valid_products:
+                if p.lower() == vp.lower():
+                    normalized_products.append(vp)
+                    break
+
+        # Validate competitors
+        competitors = result.get("competitors_mentioned", [])
+        if not isinstance(competitors, list):
+            competitors = []
+        valid_competitors = [
+            c for c in competitors
+            if any(c.lower() == vc.lower() for vc in self.valid_competitors)
+        ]
+        # Normalize to canonical case
+        normalized_competitors = []
+        for c in valid_competitors:
+            for vc in self.valid_competitors:
+                if c.lower() == vc.lower():
+                    normalized_competitors.append(vc)
+                    break
+
+        # Validate author_role
+        author_role = result.get("author_role", "unknown")
+        if author_role not in self.valid_author_roles:
+            author_role = "unknown"
+
+        # Validate sabian_mention_context
+        mention_context = result.get("sabian_mention_context")
+        is_relevant = result.get("is_relevant", False)
+
+        if not is_relevant:
+            mention_context = None
+        elif mention_context and mention_context not in self.valid_mention_contexts:
+            mention_context = "casual_mention"  # Default for relevant posts
+
+        # Validate confidence
+        confidence = result.get("relevance_confidence", "medium")
+        if confidence not in ["high", "medium", "low"]:
+            confidence = "medium"
+
+        return {
+            "is_relevant": bool(is_relevant),
+            "relevance_confidence": confidence,
+            "relevance_reason": result.get("relevance_reason", ""),
+            "products_mentioned": normalized_products,
+            "sabian_mention_context": mention_context,
+            "author_role": author_role,
+            "competitors_mentioned": normalized_competitors,
+            "thread_context_summary": result.get("thread_context_summary", "")
+        }
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a post through relevance validation and fact extraction.
+
+        Args:
+            input_data: Dictionary from preprocessor containing:
+                - cleaned_content: Cleaned post text
+                - quoted_content: Quoted content if any
+                - raw_thread_context: Raw thread context
+                - relevance_keywords_found: Keywords from preprocessor
+                - preliminary_relevant: Preprocessor's relevance assessment
+                - needs_relevance_validation: Whether LLM validation needed
+
+        Returns:
+            Dictionary with extraction results and original data
+        """
+        try:
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    "is_relevant": False,
+                    **input_data
+                }
+
+            # Skip if already determined not relevant and no validation needed
+            if (not input_data.get("preliminary_relevant", False) and
+                not input_data.get("needs_relevance_validation", False)):
+                return {
+                    "success": True,
+                    "is_relevant": False,
+                    "relevance_confidence": "high",
+                    "relevance_reason": "No Sabian-related keywords found in post",
+                    "products_mentioned": [],
+                    "sabian_mention_context": None,
+                    "author_role": "unknown",
+                    "competitors_mentioned": input_data.get("competitors_detected", []),
+                    "thread_context_summary": "",
+                    "extraction_performed": False,
+                    **input_data
+                }
+
+            # Skip non-English posts
+            if not input_data.get("is_english", True):
+                return {
+                    "success": True,
+                    "is_relevant": False,
+                    "relevance_confidence": "high",
+                    "relevance_reason": f"Non-English post: {input_data.get('detected_language')}",
+                    "products_mentioned": [],
+                    "sabian_mention_context": None,
+                    "author_role": "unknown",
+                    "competitors_mentioned": [],
+                    "thread_context_summary": "",
+                    "extraction_performed": False,
+                    **input_data
+                }
+
+            # Perform LLM extraction
+            extraction_result = self.extract_and_validate(input_data)
+
+            # Merge results
+            result = {
+                **input_data,
+                **extraction_result,
+                "extraction_performed": True
+            }
+
+            # Log the result
+            self.log_processing(
+                f"Extraction complete: is_relevant={result.get('is_relevant')}, "
+                f"products={result.get('products_mentioned')}, "
+                f"context={result.get('sabian_mention_context')}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "relevance extraction")

processing_brand_sentiment/workflow/agents/sabian_sentiment_analyzer_agent.py

@@ -0,0 +1,434 @@
+"""
+Sabian Sentiment & Intent Analyzer Agent for brand sentiment analysis.
+
+This agent performs deep analysis on VERIFIED relevant posts with STRUCTURED input.
+It receives pre-validated data from the Relevance Extraction Agent including:
+- Products already extracted and validated
+- Thread context already summarized
+- Author role already determined
+
+Key Design Principles:
+- Focused analysis: Only sentiment, intents, and customer journey
+- No re-extraction: Products are given, not re-detected
+- Sabian-specific sentiment: How author feels about Sabian, not overall post tone
+- Author perspective: Pain points/delights only from author's own experience
+"""
+
+from typing import Dict, Any, List
+import json
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+import logging
+
+from .base_agent import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class SabianSentimentAnalyzerAgent(BaseAgent):
+    """
+    Agent that performs deep sentiment and intent analysis on relevant posts.
+
+    This agent is the second LLM call in the pipeline and focuses purely on
+    analysis, not extraction. It receives structured input from the extraction
+    agent and produces sentiment, intent, and customer journey insights.
+    """
+
+    def __init__(
+        self,
+        config: Dict[str, Any],
+        api_key: str,
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any]
+    ):
+        """
+        Initialize the Sentiment Analyzer Agent.
+
+        Args:
+            config: Agent configuration
+            api_key: OpenAI API key
+            brand_config: Brand-specific configuration
+            analysis_categories: Category definitions for analysis
+        """
+        super().__init__("SabianSentimentAnalyzerAgent", config)
+        self.api_key = api_key
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key
+        )
+
+        # Pre-compute valid values for validation
+        self._valid_values = self._compute_valid_values()
+
+        logger.info("SabianSentimentAnalyzerAgent initialized")
+
+    def _compute_valid_values(self) -> Dict[str, List[str]]:
+        """Pre-compute all valid values from config for validation."""
+        valid = {}
+
+        # Products from brand config
+        valid["products"] = self.brand_config.get("brand", {}).get("products", [])
+
+        # Competitors
+        competitor_names = []
+        for comp in self.brand_config.get("brand", {}).get("competitors", []):
+            if isinstance(comp, dict):
+                competitor_names.append(comp.get("name", ""))
+        valid["competitors"] = competitor_names
+
+        # Extract category values from analysis_categories
+        category_map = {
+            "sentiment_level": "sentiment",
+            "emotion_type": "emotions",
+            "intents": "intents",
+            "purchase_stage": "purchase_stage",
+            "comparison_type": "comparison_type",
+            "feedback_aspects": "feedback_aspects",
+            "decision_drivers": "decision_drivers",
+            "product_attributes": "product_attributes",
+        }
+
+        for key, config_key in category_map.items():
+            config_section = self.analysis_categories.get(config_key, {})
+            if "categories" in config_section:
+                valid[key] = [c["value"] for c in config_section["categories"]]
+            elif "levels" in config_section:
+                valid[key] = [c["value"] for c in config_section["levels"]]
+            else:
+                valid[key] = []
+
+        return valid
+
+    def _get_valid_list(self, key: str) -> List[str]:
+        """Get list of valid values for a category."""
+        return self._valid_values.get(key, [])
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """Validate that input contains required fields."""
+        required = ["cleaned_content", "is_relevant"]
+        return all(field in input_data for field in required)
+
+    def _build_system_prompt(self) -> str:
+        """Build optimized system prompt for sentiment analysis."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+        v = self._valid_values
+
+        return f"""You are a sentiment analyst for {brand_name} cymbal discussions.
+
+## YOUR TASK
+Analyze the sentiment, emotions, and intents in posts about {brand_name}.
+You will receive PRE-VALIDATED context (products, author role, etc.) - trust these values.
+
+## CRITICAL RULES
+
+### Rule 1: Neutral by Default
+Sentiment defaults to NEUTRAL unless there is EXPLICIT positive or negative language toward {brand_name}.
+- Factual statements = neutral
+- Comparative statements ("sounds different", "not the same as") = neutral (different ≠ worse)
+- Advice-giving without personal opinion = neutral
+
+Only assign positive/negative sentiment when the author CLEARLY expresses satisfaction or dissatisfaction with {brand_name}.
+
+### Rule 2: {brand_name}-Specific Sentiment
+Sentiment MUST be about {brand_name} specifically, NOT overall post tone or other products.
+
+EXAMPLE:
+Post: "I have SBR cymbals and bought a Pearl crash. The Pearl sounds different from the SBR. Go with what feels best!"
+- This is NEUTRAL toward {brand_name} - "different" is not criticism
+- The author owns SBR (no complaint), is giving advice
+- pain_points: [] (no negative experience expressed)
+- delight_factors: [] (no positive experience expressed)
+
+### Rule 3: Mutually Exclusive Feedback
+pain_points and delight_factors CANNOT contain the same values.
+- If an aspect is positive → delight_factors only
+- If an aspect is negative → pain_points only
+- Never both
+
+### Rule 4: Author Perspective Only
+These fields are ONLY for author's OWN experience, not advice to others:
+- purchase_stage, decision_drivers, pain_points, delight_factors
+
+If author is primarily giving ADVICE to someone else, these should be null/empty.
+
+### Rule 5: Valid Values
+
+| Field | Valid Values |
+|-------|--------------|
+| sentiment_level | {v.get('sentiment_level', [])} |
+| emotion_type | {v.get('emotion_type', [])} |
+| intents (multi-select) | {v.get('intents', [])} |
+| purchase_stage | {v.get('purchase_stage', [])} |
+| comparison_type | {v.get('comparison_type', [])} |
+| feedback_aspects | {v.get('feedback_aspects', [])} |
+| decision_drivers | {v.get('decision_drivers', [])} |
+| product_attributes | {v.get('product_attributes', [])} |
+| competitor brands | {v.get('competitors', [])} |
+
+### Rule 6: Intent Classification
+- seeking_information: Asking questions, seeking advice
+- providing_information: Answering questions, giving advice
+- sharing_experience: Personal experience, review, testimonial
+- comparing: Comparing brands/products
+- praising: Actively endorsing {brand_name}
+- criticizing: Actively complaining about {brand_name}
+- buying_selling: Listing gear for sale/trade
+- general_discussion: General conversation
+
+## OUTPUT FORMAT
+```json
+{{
+    "sentiment_level": "neutral unless explicit positive/negative",
+    "emotion_type": "value or null",
+    "sentiment_confidence": "high" | "medium" | "low",
+    "sarcasm_detected": false,
+    "product_attributes": [],
+    "competitor_products_owned": [],
+    "comparison_type": "value or null",
+    "intents": [],
+    "purchase_stage": "value or null",
+    "decision_drivers": [],
+    "pain_points": [],
+    "delight_factors": [],
+    "analysis_notes": "1-2 sentences"
+}}
+```
+
+Return ONLY valid JSON."""
+
+    def _build_user_prompt(self, input_data: Dict[str, Any]) -> str:
+        """Build user prompt with structured context."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+
+        content = input_data.get("cleaned_content", "")
+        products_mentioned = input_data.get("products_mentioned", [])
+        sabian_context = input_data.get("sabian_mention_context", "")
+        author_role = input_data.get("author_role", "unknown")
+        thread_summary = input_data.get("thread_context_summary", "")
+        competitors_mentioned = input_data.get("competitors_mentioned", [])
+
+        context_section = f"""## PRE-VALIDATED CONTEXT (trust these values)
+- Products mentioned: {products_mentioned if products_mentioned else 'None specific'}
+- {brand_name} mention context: {sabian_context}
+- Author role: {author_role}
+- Competitors mentioned: {competitors_mentioned if competitors_mentioned else 'None'}
+- Thread summary: {thread_summary if thread_summary else 'Not available'}
+"""
+
+        return f"""Analyze this post about {brand_name} for sentiment and intents.
+
+{context_section}
+## POST CONTENT TO ANALYZE:
+\"\"\"{content}\"\"\"
+
+Remember:
+- Sentiment is about {brand_name} ONLY, not overall post tone
+- pain_points/delight_factors only from author's OWN experience
+- Use only values from the valid lists provided
+
+Return JSON only."""
+
+    def analyze_post(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Perform sentiment and intent analysis.
+
+        Args:
+            input_data: Structured data from extraction agent
+
+        Returns:
+            Dictionary with analysis results
+        """
+        try:
+            messages = [
+                SystemMessage(content=self._build_system_prompt()),
+                HumanMessage(content=self._build_user_prompt(input_data))
+            ]
+
+            response = self.llm.invoke(messages)
+            result = self._parse_llm_json_response(response.content)
+
+            # Validate and normalize
+            validated = self._validate_and_normalize(result)
+
+            return {"success": True, **validated}
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"JSON decode error: {e}", "warning")
+            return {
+                "success": False,
+                "error": f"JSON parse error: {str(e)}",
+                "sentiment_level": "neutral",
+                "intents": ["general_discussion"]
+            }
+
+        except Exception as e:
+            self.log_processing(f"Analysis error: {e}", "error")
+            return {"success": False, "error": str(e)}
+
+    def _validate_single(self, value: Any, valid_list: List[str], default: Any = None) -> Any:
+        """Validate single value against list, return canonical form or default."""
+        if value is None:
+            return default
+        if isinstance(value, str):
+            val_lower = value.lower()
+            for v in valid_list:
+                if v.lower() == val_lower:
+                    return v
+        return default
+
+    def _validate_list(self, values: Any, valid_list: List[str]) -> List[str]:
+        """Validate list values, return only valid items in canonical form."""
+        if not values:
+            return []
+        if not isinstance(values, list):
+            values = [values]
+
+        validated = []
+        valid_lower = {v.lower(): v for v in valid_list}
+        for val in values:
+            if isinstance(val, str) and val.lower() in valid_lower:
+                validated.append(valid_lower[val.lower()])
+        return validated
+
+    def _validate_and_normalize(self, result: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate all fields against predefined values and normalize."""
+        v = self._valid_values
+
+        normalized = {
+            # Sentiment
+            "sentiment_level": self._validate_single(
+                result.get("sentiment_level"), v["sentiment_level"], "neutral"
+            ),
+            "emotion_type": self._validate_single(
+                result.get("emotion_type"), v["emotion_type"], None
+            ),
+            "sentiment_confidence": result.get("sentiment_confidence", "medium"),
+            "sarcasm_detected": bool(result.get("sarcasm_detected", False)),
+
+            # Product info
+            "product_attributes": self._validate_list(
+                result.get("product_attributes"), v["product_attributes"]
+            ),
+
+            # Competitors
+            "competitor_products_owned": self._validate_list(
+                result.get("competitor_products_owned"), v["competitors"]
+            ),
+            "comparison_type": self._validate_single(
+                result.get("comparison_type"), v["comparison_type"], None
+            ),
+
+            # Intents
+            "intents": self._validate_list(
+                result.get("intents"), v["intents"]
+            ) or ["general_discussion"],
+
+            # Author journey (null if advising others)
+            "purchase_stage": self._validate_single(
+                result.get("purchase_stage"), v["purchase_stage"], None
+            ),
+            "decision_drivers": self._validate_list(
+                result.get("decision_drivers"), v["decision_drivers"]
+            ),
+
+            # Feedback - both use feedback_aspects
+            "pain_points": self._validate_list(
+                result.get("pain_points"), v["feedback_aspects"]
+            ),
+            "delight_factors": self._validate_list(
+                result.get("delight_factors"), v["feedback_aspects"]
+            ),
+
+            # Notes
+            "analysis_notes": result.get("analysis_notes", ""),
+        }
+
+        # Validate confidence
+        if normalized["sentiment_confidence"] not in ["high", "medium", "low"]:
+            normalized["sentiment_confidence"] = "medium"
+
+        return normalized
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a post through sentiment and intent analysis.
+
+        Args:
+            input_data: Dictionary from extraction agent containing:
+                - cleaned_content: Post text
+                - is_relevant: Relevance determination
+                - products_mentioned: Pre-validated products
+                - sabian_mention_context: How Sabian is discussed
+                - author_role: Author's relationship to Sabian
+                - thread_context_summary: Summarized context
+                - competitors_mentioned: Competitor brands
+
+        Returns:
+            Dictionary with analysis results and original data
+        """
+        try:
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    **input_data
+                }
+
+            # Skip non-relevant posts
+            if not input_data.get("is_relevant", False):
+                return {
+                    "success": True,
+                    "analysis_skipped": True,
+                    "analysis_skip_reason": "Post marked as not relevant",
+                    "sentiment_level": None,
+                    "emotion_type": None,
+                    "sentiment_confidence": None,
+                    "sarcasm_detected": False,
+                    "product_attributes": [],
+                    "competitor_products_owned": [],
+                    "comparison_type": None,
+                    "intents": [],
+                    "purchase_stage": None,
+                    "decision_drivers": [],
+                    "pain_points": [],
+                    "delight_factors": [],
+                    "analysis_notes": "",
+                    **input_data
+                }
+
+            # Skip non-English posts (should already be filtered, but double-check)
+            if not input_data.get("is_english", True):
+                return {
+                    "success": True,
+                    "analysis_skipped": True,
+                    "analysis_skip_reason": f"Non-English: {input_data.get('detected_language')}",
+                    "sentiment_level": None,
+                    "emotion_type": None,
+                    "intents": [],
+                    **input_data
+                }
+
+            # Perform analysis
+            analysis_result = self.analyze_post(input_data)
+
+            result = {
+                **input_data,
+                **analysis_result,
+                "analysis_skipped": False
+            }
+
+            self.log_processing(
+                f"Analyzed: sentiment={result.get('sentiment_level')}, "
+                f"intents={result.get('intents')}, "
+                f"pain_points={result.get('pain_points')}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "sentiment analysis")

processing_brand_sentiment/workflow/comment_orchestrator.py

@@ -0,0 +1,558 @@
+"""
+Comment Analysis Workflow Orchestrator using LangGraph.
+
+Coordinates the 4-agent pipeline for social media comments:
+1. CommentPreprocessorAgent - Plain text cleaning, keyword detection (no LLM)
+2. SabianRelevanceExtractionAgent - Relevance + fact extraction (LLM #1) [shared]
+3. SabianSentimentAnalyzerAgent - Deep sentiment analysis (LLM #2) [shared]
+4. OutputValidatorAgent - Rule-based validation (no LLM) [shared]
+
+Architecture v4.0:
+- Same analysis pipeline as forums, different preprocessing and state
+- Plain text input (no HTML parsing)
+- Context from social media content metadata and parent comments
+- Comment-specific identifiers (comment_sk, comment_id, platform, etc.)
+"""
+
+from typing import Dict, Any, List, TypedDict, Annotated, Optional
+import operator
+import json
+import os
+from langgraph.graph import StateGraph, END
+import logging
+
+from .agents.comment_preprocessor_agent import CommentPreprocessorAgent
+from .agents.sabian_relevance_extraction_agent import SabianRelevanceExtractionAgent
+from .agents.sabian_sentiment_analyzer_agent import SabianSentimentAnalyzerAgent
+from .agents.output_validator_agent import OutputValidatorAgent
+
+logger = logging.getLogger(__name__)
+
+
+class CommentAnalysisState(TypedDict):
+    """
+    State definition for the comment analysis workflow v4.0.
+
+    Uses comment-specific identifiers but shares the same analysis fields
+    as the forum workflow for consistent output.
+    """
+    # ============== Source Identifiers (Comment-specific) ==============
+    comment_sk: int
+    comment_id: str
+    platform: str
+    comment_timestamp: Any
+    author_name: str
+    author_id: str
+    parent_comment_id: str
+    parent_comment_text: str
+
+    # Content metadata
+    content_sk: int
+    content_id: str
+    content_description: str
+    content_title: str
+    channel_sk: int
+    channel_name: str
+    channel_display_name: str
+
+    # ============== Original Content ==============
+    comment_text: str
+    original_text: str
+
+    # ============== Preprocessor Output ==============
+    cleaned_content: str
+    quoted_content: str
+    has_quote: bool
+    quoted_author: str
+    raw_thread_context: str  # Comment context (reuses field name for agent compatibility)
+    is_empty: bool
+
+    # Language detection
+    detected_language: str
+    language_code: str
+    is_english: bool
+    language_confidence: str
+    language_detection_skipped: bool
+
+    # Preliminary relevance (keyword-based)
+    preliminary_relevant: bool
+    needs_relevance_validation: bool
+    relevance_keywords_found: List[str]
+    relevance_type: str
+    has_primary_keywords: bool
+
+    # Initial detections
+    products_detected: List[str]
+    competitors_detected: List[str]
+
+    # ============== Extraction Agent Output ==============
+    is_relevant: bool
+    relevance_confidence: str
+    relevance_reason: str
+    extraction_performed: bool
+
+    # Extracted facts
+    products_mentioned: List[str]
+    sabian_mention_context: str
+    author_role: str
+    competitors_mentioned: List[str]
+    thread_context_summary: str
+
+    # ============== Sentiment Analyzer Output ==============
+    sentiment_level: str
+    emotion_type: str
+    sentiment_confidence: str
+    sarcasm_detected: bool
+
+    # Product information
+    product_attributes: List[str]
+
+    # Competitive intelligence
+    competitor_products_owned: List[str]
+    comparison_type: str
+
+    # Customer journey (AUTHOR PERSPECTIVE ONLY)
+    intents: List[str]
+    purchase_stage: str
+    decision_drivers: List[str]
+    pain_points: List[str]
+    delight_factors: List[str]
+
+    # Analysis notes
+    analysis_notes: str
+    analysis_skipped: bool
+    analysis_skip_reason: str
+
+    # ============== Validator Output ==============
+    validation_passed: bool
+    validation_errors: List[str]
+    validation_warnings: List[str]
+    validation_flags: List[str]
+    processing_status: str
+
+    # ============== Processing Metadata ==============
+    processing_errors: Annotated[List[str], operator.add]
+    success: bool
+
+
+class CommentAnalysisWorkflow:
+    """
+    LangGraph-based workflow for comment brand sentiment analysis v4.0.
+
+    Pipeline:
+    1. Comment Preprocessor (no LLM) - plain text, comment context
+    2. Relevance & Extraction Agent (LLM #1) - shared with forums
+    3. Sentiment Analyzer Agent (LLM #2) - shared with forums
+    4. Output Validator (no LLM) - shared with forums
+    """
+
+    def __init__(
+        self,
+        workflow_config: Dict[str, Any],
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any],
+        api_key: str
+    ):
+        """
+        Initialize the workflow with agents and configuration.
+
+        Args:
+            workflow_config: Workflow and agent configuration
+            brand_config: Brand-specific configuration
+            analysis_categories: Analysis category definitions
+            api_key: OpenAI API key
+        """
+        self.workflow_config = workflow_config
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+        self.api_key = api_key
+
+        # Initialize agents
+        self._init_agents()
+
+        # Build the workflow graph
+        self.workflow = self._build_workflow()
+
+        logger.info("CommentAnalysisWorkflow v4.0 initialized successfully")
+
+    def _init_agents(self) -> None:
+        """Initialize all agents with their configurations."""
+        agents_config = self.workflow_config.get("agents", {})
+
+        # 1. Comment Preprocessor Agent (no LLM) - comment-specific
+        preprocessor_config = agents_config.get("preprocessor", {})
+        self.preprocessor = CommentPreprocessorAgent(
+            preprocessor_config,
+            self.brand_config
+        )
+
+        # 2. Relevance & Extraction Agent (LLM #1) - shared with forums
+        extraction_config = agents_config.get("relevance_extraction",
+            agents_config.get("relevance_validator", {})
+        )
+        self.extraction_agent = SabianRelevanceExtractionAgent(
+            extraction_config,
+            self.api_key,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        # 3. Sentiment Analyzer Agent (LLM #2) - shared with forums
+        analyzer_config = agents_config.get("sentiment_analyzer",
+            agents_config.get("brand_analyzer", {})
+        )
+        self.sentiment_analyzer = SabianSentimentAnalyzerAgent(
+            analyzer_config,
+            self.api_key,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        # 4. Output Validator Agent (no LLM) - shared with forums
+        validator_config = agents_config.get("output_validator", {})
+        self.output_validator = OutputValidatorAgent(
+            validator_config,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        logger.info("All 4 agents initialized for comment processing")
+
+    def _build_workflow(self) -> StateGraph:
+        """
+        Build the LangGraph workflow.
+
+        Flow:
+        preprocessing -> extraction -> (analysis if relevant) -> validation -> END
+
+        Returns:
+            Compiled StateGraph workflow
+        """
+        workflow = StateGraph(CommentAnalysisState)
+
+        # Add nodes
+        workflow.add_node("preprocessing", self._preprocessing_node)
+        workflow.add_node("extraction", self._extraction_node)
+        workflow.add_node("analysis", self._analysis_node)
+        workflow.add_node("validation", self._validation_node)
+
+        # Set entry point
+        workflow.set_entry_point("preprocessing")
+
+        # Define edges
+        workflow.add_conditional_edges(
+            "preprocessing",
+            self._route_after_preprocessing,
+            {
+                "extract": "extraction",
+                "skip_to_validation": "validation"
+            }
+        )
+
+        workflow.add_conditional_edges(
+            "extraction",
+            self._route_after_extraction,
+            {
+                "analyze": "analysis",
+                "skip_to_validation": "validation"
+            }
+        )
+
+        workflow.add_edge("analysis", "validation")
+        workflow.add_edge("validation", END)
+
+        return workflow.compile()
+
+    def _preprocessing_node(self, state: CommentAnalysisState) -> CommentAnalysisState:
+        """
+        Preprocessing node: Plain text cleaning, language detection, keyword check.
+        """
+        try:
+            input_data = {
+                "comment_sk": state.get("comment_sk"),
+                "comment_text": state.get("comment_text", ""),
+                "content_title": state.get("content_title"),
+                "content_description": state.get("content_description"),
+                "parent_comment_text": state.get("parent_comment_text")
+            }
+
+            result = self.preprocessor.process(input_data)
+
+            if result.get("success", False):
+                # Content
+                state["cleaned_content"] = result.get("cleaned_content", "")
+                state["quoted_content"] = result.get("quoted_content")
+                state["has_quote"] = result.get("has_quote", False)
+                state["quoted_author"] = result.get("quoted_author")
+                state["raw_thread_context"] = result.get("raw_thread_context", "")
+                state["is_empty"] = result.get("is_empty", False)
+                state["original_text"] = result.get("original_text", state.get("comment_text", ""))
+
+                # Language
+                state["detected_language"] = result.get("detected_language", "English")
+                state["language_code"] = result.get("language_code", "en")
+                state["is_english"] = result.get("is_english", True)
+                state["language_confidence"] = result.get("language_confidence", "low")
+                state["language_detection_skipped"] = result.get("language_detection_skipped", False)
+
+                # Relevance
+                state["preliminary_relevant"] = result.get("preliminary_relevant", False)
+                state["needs_relevance_validation"] = result.get("needs_relevance_validation", False)
+                state["relevance_keywords_found"] = result.get("relevance_keywords_found", [])
+                state["relevance_type"] = result.get("relevance_type", "none")
+                state["has_primary_keywords"] = result.get("has_primary_keywords", False)
+
+                # Detections
+                state["products_detected"] = result.get("products_detected", [])
+                state["competitors_detected"] = result.get("competitors_detected", [])
+
+                state["success"] = True
+            else:
+                error_msg = f"Preprocessing failed: {result.get('error', 'Unknown error')}"
+                state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+                state["success"] = False
+
+            logger.debug(f"Preprocessing complete for comment {state.get('comment_sk')}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Preprocessing node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["success"] = False
+            return state
+
+    def _extraction_node(self, state: CommentAnalysisState) -> CommentAnalysisState:
+        """
+        Extraction node: LLM-based relevance validation and fact extraction.
+        Reuses the same extraction agent as forums.
+        """
+        try:
+            input_data = {
+                "cleaned_content": state.get("cleaned_content", ""),
+                "quoted_content": state.get("quoted_content"),
+                "raw_thread_context": state.get("raw_thread_context", ""),
+                "relevance_keywords_found": state.get("relevance_keywords_found", []),
+                "preliminary_relevant": state.get("preliminary_relevant", False),
+                "needs_relevance_validation": state.get("needs_relevance_validation", True),
+                "products_detected": state.get("products_detected", []),
+                "competitors_detected": state.get("competitors_detected", []),
+                "is_english": state.get("is_english", True),
+                "detected_language": state.get("detected_language", "English")
+            }
+
+            result = self.extraction_agent.process(input_data)
+
+            # Update state with extraction results
+            state["is_relevant"] = result.get("is_relevant", False)
+            state["relevance_confidence"] = result.get("relevance_confidence", "low")
+            state["relevance_reason"] = result.get("relevance_reason", "")
+            state["extraction_performed"] = result.get("extraction_performed", True)
+
+            # Extracted facts
+            state["products_mentioned"] = result.get("products_mentioned", [])
+            state["sabian_mention_context"] = result.get("sabian_mention_context")
+            state["author_role"] = result.get("author_role", "unknown")
+            state["competitors_mentioned"] = result.get("competitors_mentioned", [])
+            state["thread_context_summary"] = result.get("thread_context_summary", "")
+
+            if not result.get("success", False) and result.get("error"):
+                state["processing_errors"] = state.get("processing_errors", []) + [result["error"]]
+
+            logger.debug(f"Extraction complete: is_relevant={state['is_relevant']}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Extraction node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["is_relevant"] = False
+            state["relevance_confidence"] = "low"
+            return state
+
+    def _analysis_node(self, state: CommentAnalysisState) -> CommentAnalysisState:
+        """
+        Analysis node: Deep sentiment and intent analysis for relevant comments.
+        Reuses the same sentiment analyzer as forums.
+        """
+        try:
+            input_data = {
+                "cleaned_content": state.get("cleaned_content", ""),
+                "is_relevant": state.get("is_relevant", True),
+                "is_english": state.get("is_english", True),
+                "detected_language": state.get("detected_language", "English"),
+                "products_mentioned": state.get("products_mentioned", []),
+                "sabian_mention_context": state.get("sabian_mention_context"),
+                "author_role": state.get("author_role", "unknown"),
+                "competitors_mentioned": state.get("competitors_mentioned", []),
+                "thread_context_summary": state.get("thread_context_summary", "")
+            }
+
+            result = self.sentiment_analyzer.process(input_data)
+
+            if result.get("success", False):
+                # Sentiment
+                state["sentiment_level"] = result.get("sentiment_level")
+                state["emotion_type"] = result.get("emotion_type")
+                state["sentiment_confidence"] = result.get("sentiment_confidence", "medium")
+                state["sarcasm_detected"] = result.get("sarcasm_detected", False)
+
+                # Products
+                state["product_attributes"] = result.get("product_attributes", [])
+
+                # Competitive
+                state["competitor_products_owned"] = result.get("competitor_products_owned", [])
+                state["comparison_type"] = result.get("comparison_type")
+
+                # Journey
+                state["intents"] = result.get("intents", [])
+                state["purchase_stage"] = result.get("purchase_stage")
+                state["decision_drivers"] = result.get("decision_drivers", [])
+                state["pain_points"] = result.get("pain_points", [])
+                state["delight_factors"] = result.get("delight_factors", [])
+
+                # Notes
+                state["analysis_notes"] = result.get("analysis_notes", "")
+                state["analysis_skipped"] = result.get("analysis_skipped", False)
+                state["analysis_skip_reason"] = result.get("analysis_skip_reason", "")
+            else:
+                error_msg = f"Analysis failed: {result.get('error', 'Unknown error')}"
+                state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+
+            logger.debug(f"Analysis complete for comment {state.get('comment_sk')}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Analysis node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            return state
+
+    def _validation_node(self, state: CommentAnalysisState) -> CommentAnalysisState:
+        """
+        Validation node: Rule-based validation and anomaly detection.
+        Reuses the same output validator as forums.
+        """
+        try:
+            result = self.output_validator.process(dict(state))
+
+            state["validation_passed"] = result.get("validation_passed", True)
+            state["validation_errors"] = result.get("validation_errors", [])
+            state["validation_warnings"] = result.get("validation_warnings", [])
+            state["validation_flags"] = result.get("validation_flags", [])
+            state["processing_status"] = result.get("processing_status", "completed")
+
+            # Set overall success
+            has_errors = len(state.get("processing_errors", [])) > 0
+            state["success"] = not has_errors or state.get("is_relevant") is not None
+
+            logger.debug(f"Validation complete: status={state['processing_status']}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Validation node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["validation_passed"] = False
+            state["processing_status"] = "validation_failed"
+            state["success"] = False
+            return state
+
+    def _route_after_preprocessing(self, state: CommentAnalysisState) -> str:
+        """Determine routing after preprocessing."""
+        if state.get("is_empty", False):
+            state["is_relevant"] = False
+            state["relevance_reason"] = "Empty content"
+            return "skip_to_validation"
+
+        if not state.get("is_english", True):
+            state["is_relevant"] = False
+            state["relevance_reason"] = f"Non-English: {state.get('detected_language')}"
+            return "skip_to_validation"
+
+        if (not state.get("preliminary_relevant", False) and
+            not state.get("needs_relevance_validation", False)):
+            state["is_relevant"] = False
+            state["relevance_reason"] = "No relevant keywords found"
+            return "skip_to_validation"
+
+        return "extract"
+
+    def _route_after_extraction(self, state: CommentAnalysisState) -> str:
+        """Determine routing after extraction."""
+        if state.get("is_relevant", False):
+            return "analyze"
+        return "skip_to_validation"
+
+    def process_comment(self, comment_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a single social media comment through the workflow.
+
+        Args:
+            comment_data: Dictionary containing comment data
+
+        Returns:
+            Dictionary with processed results
+        """
+        try:
+            initial_state = {
+                # Comment identifiers
+                "comment_sk": comment_data.get("comment_sk"),
+                "comment_id": comment_data.get("comment_id"),
+                "platform": comment_data.get("platform"),
+                "comment_timestamp": comment_data.get("comment_timestamp"),
+                "author_name": comment_data.get("author_name"),
+                "author_id": comment_data.get("author_id"),
+                "parent_comment_id": comment_data.get("parent_comment_id"),
+                "parent_comment_text": comment_data.get("parent_comment_text"),
+
+                # Content metadata
+                "content_sk": comment_data.get("content_sk"),
+                "content_id": comment_data.get("content_id"),
+                "content_description": comment_data.get("content_description"),
+                "content_title": comment_data.get("content_title"),
+                "channel_sk": comment_data.get("channel_sk"),
+                "channel_name": comment_data.get("channel_name"),
+                "channel_display_name": comment_data.get("channel_display_name"),
+
+                # Comment text
+                "comment_text": comment_data.get("comment_text", ""),
+
+                # Processing metadata
+                "processing_errors": [],
+                "success": True
+            }
+
+            final_state = self.workflow.invoke(initial_state)
+
+            return dict(final_state)
+
+        except Exception as e:
+            logger.error(f"Workflow execution error: {str(e)}")
+            return {
+                **comment_data,
+                "success": False,
+                "processing_errors": [str(e)],
+                "processing_status": "workflow_error"
+            }
+
+    def process_batch(self, comments: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Process a batch of social media comments.
+
+        Args:
+            comments: List of comment dictionaries
+
+        Returns:
+            List of processed comment dictionaries
+        """
+        results = []
+        total = len(comments)
+
+        for idx, comment in enumerate(comments, 1):
+            logger.info(f"Processing comment {idx}/{total} (SK: {comment.get('comment_sk')})")
+            result = self.process_comment(comment)
+            results.append(result)
+
+        logger.info(f"Batch processing complete: {total} comments processed")
+        return results

processing_brand_sentiment/workflow/orchestrator.py

@@ -0,0 +1,551 @@
+"""
+Brand Analysis Workflow Orchestrator using LangGraph.
+
+Coordinates the 4-agent pipeline:
+1. ContentPreprocessorAgent - HTML parsing, cleaning, keyword detection (no LLM)
+2. SabianRelevanceExtractionAgent - Relevance + fact extraction (LLM #1)
+3. SabianSentimentAnalyzerAgent - Deep sentiment analysis (LLM #2)
+4. OutputValidatorAgent - Rule-based validation (no LLM)
+
+Architecture v4.0:
+- Separation of concerns: extraction vs analysis
+- Strict validation at every step
+- Structured data flow between agents
+- Conservative relevance determination
+"""
+
+from typing import Dict, Any, List, TypedDict, Annotated, Optional
+import operator
+import json
+import os
+from langgraph.graph import StateGraph, END
+import logging
+
+from .agents.content_preprocessor_agent import ContentPreprocessorAgent
+from .agents.sabian_relevance_extraction_agent import SabianRelevanceExtractionAgent
+from .agents.sabian_sentiment_analyzer_agent import SabianSentimentAnalyzerAgent
+from .agents.output_validator_agent import OutputValidatorAgent
+
+logger = logging.getLogger(__name__)
+
+
+class BrandAnalysisState(TypedDict):
+    """
+    State definition for the brand analysis workflow v4.0.
+
+    This state flows through all agents, accumulating data at each step.
+    """
+    # ============== Source Identifiers ==============
+    post_id: int
+    thread_id: int
+    post_author_id: int
+
+    # ============== Original Content ==============
+    post_content: str
+    original_content: str
+
+    # ============== Thread Context ==============
+    thread_title: str
+    thread_first_post: str
+    thread_started_at: Any
+    category_title: str
+    category_topic: str
+
+    # ============== Timestamps ==============
+    post_created_at: Any
+
+    # ============== Preprocessor Output ==============
+    cleaned_content: str
+    quoted_content: str
+    has_quote: bool
+    quoted_author: str
+    raw_thread_context: str  # Raw context for extraction agent
+    is_empty: bool
+
+    # Language detection
+    detected_language: str
+    language_code: str
+    is_english: bool
+    language_confidence: str
+    language_detection_skipped: bool
+
+    # Preliminary relevance (keyword-based)
+    preliminary_relevant: bool
+    needs_relevance_validation: bool
+    relevance_keywords_found: List[str]
+    relevance_type: str
+    has_primary_keywords: bool
+
+    # Initial detections
+    products_detected: List[str]
+    competitors_detected: List[str]
+
+    # ============== Extraction Agent Output ==============
+    is_relevant: bool
+    relevance_confidence: str
+    relevance_reason: str
+    extraction_performed: bool
+
+    # Extracted facts
+    products_mentioned: List[str]
+    sabian_mention_context: str  # primary_focus, significant_mention, casual_mention, comparison_context
+    author_role: str  # current_owner, past_owner, potential_buyer, never_owned, unknown
+    competitors_mentioned: List[str]
+    thread_context_summary: str  # NEW: Summarized context for storage and analysis
+
+    # ============== Sentiment Analyzer Output ==============
+    sentiment_level: str
+    emotion_type: str
+    sentiment_confidence: str
+    sarcasm_detected: bool
+
+    # Product information
+    product_attributes: List[str]
+
+    # Competitive intelligence
+    competitor_products_owned: List[str]
+    comparison_type: str
+
+    # Customer journey (AUTHOR PERSPECTIVE ONLY)
+    intents: List[str]
+    purchase_stage: str
+    decision_drivers: List[str]
+    pain_points: List[str]
+    delight_factors: List[str]
+
+    # Analysis notes
+    analysis_notes: str
+    analysis_skipped: bool
+    analysis_skip_reason: str
+
+    # ============== Validator Output ==============
+    validation_passed: bool
+    validation_errors: List[str]
+    validation_warnings: List[str]
+    validation_flags: List[str]
+    processing_status: str  # completed, completed_with_flags, validation_failed
+
+    # ============== Processing Metadata ==============
+    processing_errors: Annotated[List[str], operator.add]
+    success: bool
+
+
+class BrandAnalysisWorkflow:
+    """
+    LangGraph-based workflow for brand sentiment analysis v4.0.
+
+    Pipeline:
+    1. Content Preprocessor (no LLM)
+    2. Relevance & Extraction Agent (LLM #1)
+    3. Sentiment Analyzer Agent (LLM #2) - only for relevant posts
+    4. Output Validator (no LLM)
+    """
+
+    def __init__(
+        self,
+        workflow_config: Dict[str, Any],
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any],
+        api_key: str
+    ):
+        """
+        Initialize the workflow with agents and configuration.
+
+        Args:
+            workflow_config: Workflow and agent configuration
+            brand_config: Brand-specific configuration
+            analysis_categories: Analysis category definitions
+            api_key: OpenAI API key
+        """
+        self.workflow_config = workflow_config
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+        self.api_key = api_key
+
+        # Initialize agents
+        self._init_agents()
+
+        # Build the workflow graph
+        self.workflow = self._build_workflow()
+
+        logger.info("BrandAnalysisWorkflow v4.0 initialized successfully")
+
+    def _init_agents(self) -> None:
+        """Initialize all agents with their configurations."""
+        agents_config = self.workflow_config.get("agents", {})
+
+        # 1. Content Preprocessor Agent (no LLM)
+        preprocessor_config = agents_config.get("preprocessor", {})
+        self.preprocessor = ContentPreprocessorAgent(
+            preprocessor_config,
+            self.brand_config
+        )
+
+        # 2. Relevance & Extraction Agent (LLM #1)
+        extraction_config = agents_config.get("relevance_extraction",
+            agents_config.get("relevance_validator", {})  # Fallback to old config
+        )
+        self.extraction_agent = SabianRelevanceExtractionAgent(
+            extraction_config,
+            self.api_key,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        # 3. Sentiment Analyzer Agent (LLM #2)
+        analyzer_config = agents_config.get("sentiment_analyzer",
+            agents_config.get("brand_analyzer", {})  # Fallback to old config
+        )
+        self.sentiment_analyzer = SabianSentimentAnalyzerAgent(
+            analyzer_config,
+            self.api_key,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        # 4. Output Validator Agent (no LLM)
+        validator_config = agents_config.get("output_validator", {})
+        self.output_validator = OutputValidatorAgent(
+            validator_config,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        logger.info("All 4 agents initialized")
+
+    def _build_workflow(self) -> StateGraph:
+        """
+        Build the LangGraph workflow.
+
+        Flow:
+        preprocessing -> extraction -> (analysis if relevant) -> validation -> END
+
+        Returns:
+            Compiled StateGraph workflow
+        """
+        workflow = StateGraph(BrandAnalysisState)
+
+        # Add nodes
+        workflow.add_node("preprocessing", self._preprocessing_node)
+        workflow.add_node("extraction", self._extraction_node)
+        workflow.add_node("analysis", self._analysis_node)
+        workflow.add_node("validation", self._validation_node)
+
+        # Set entry point
+        workflow.set_entry_point("preprocessing")
+
+        # Define edges
+        # Preprocessing -> conditional routing
+        workflow.add_conditional_edges(
+            "preprocessing",
+            self._route_after_preprocessing,
+            {
+                "extract": "extraction",
+                "skip_to_validation": "validation"
+            }
+        )
+
+        # Extraction -> conditional routing
+        workflow.add_conditional_edges(
+            "extraction",
+            self._route_after_extraction,
+            {
+                "analyze": "analysis",
+                "skip_to_validation": "validation"
+            }
+        )
+
+        # Analysis -> validation
+        workflow.add_edge("analysis", "validation")
+
+        # Validation -> END
+        workflow.add_edge("validation", END)
+
+        return workflow.compile()
+
+    def _preprocessing_node(self, state: BrandAnalysisState) -> BrandAnalysisState:
+        """
+        Preprocessing node: HTML parsing, cleaning, language detection, keyword check.
+        """
+        try:
+            input_data = {
+                "post_id": state.get("post_id"),
+                "post_content": state.get("post_content", ""),
+                "thread_title": state.get("thread_title"),
+                "thread_first_post": state.get("thread_first_post"),
+                "category_title": state.get("category_title"),
+                "category_topic": state.get("category_topic")
+            }
+
+            result = self.preprocessor.process(input_data)
+
+            if result.get("success", False):
+                # Content
+                state["cleaned_content"] = result.get("cleaned_content", "")
+                state["quoted_content"] = result.get("quoted_content")
+                state["has_quote"] = result.get("has_quote", False)
+                state["quoted_author"] = result.get("quoted_author")
+                state["raw_thread_context"] = result.get("raw_thread_context", "")
+                state["is_empty"] = result.get("is_empty", False)
+                state["original_content"] = result.get("original_content", state.get("post_content", ""))
+
+                # Language
+                state["detected_language"] = result.get("detected_language", "English")
+                state["language_code"] = result.get("language_code", "en")
+                state["is_english"] = result.get("is_english", True)
+                state["language_confidence"] = result.get("language_confidence", "low")
+                state["language_detection_skipped"] = result.get("language_detection_skipped", False)
+
+                # Relevance
+                state["preliminary_relevant"] = result.get("preliminary_relevant", False)
+                state["needs_relevance_validation"] = result.get("needs_relevance_validation", False)
+                state["relevance_keywords_found"] = result.get("relevance_keywords_found", [])
+                state["relevance_type"] = result.get("relevance_type", "none")
+                state["has_primary_keywords"] = result.get("has_primary_keywords", False)
+
+                # Detections
+                state["products_detected"] = result.get("products_detected", [])
+                state["competitors_detected"] = result.get("competitors_detected", [])
+
+                state["success"] = True
+            else:
+                error_msg = f"Preprocessing failed: {result.get('error', 'Unknown error')}"
+                state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+                state["success"] = False
+
+            logger.debug(f"Preprocessing complete for post {state.get('post_id')}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Preprocessing node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["success"] = False
+            return state
+
+    def _extraction_node(self, state: BrandAnalysisState) -> BrandAnalysisState:
+        """
+        Extraction node: LLM-based relevance validation and fact extraction.
+        """
+        try:
+            input_data = {
+                "cleaned_content": state.get("cleaned_content", ""),
+                "quoted_content": state.get("quoted_content"),
+                "raw_thread_context": state.get("raw_thread_context", ""),
+                "relevance_keywords_found": state.get("relevance_keywords_found", []),
+                "preliminary_relevant": state.get("preliminary_relevant", False),
+                "needs_relevance_validation": state.get("needs_relevance_validation", True),
+                "products_detected": state.get("products_detected", []),
+                "competitors_detected": state.get("competitors_detected", []),
+                "is_english": state.get("is_english", True),
+                "detected_language": state.get("detected_language", "English")
+            }
+
+            result = self.extraction_agent.process(input_data)
+
+            # Update state with extraction results
+            state["is_relevant"] = result.get("is_relevant", False)
+            state["relevance_confidence"] = result.get("relevance_confidence", "low")
+            state["relevance_reason"] = result.get("relevance_reason", "")
+            state["extraction_performed"] = result.get("extraction_performed", True)
+
+            # Extracted facts
+            state["products_mentioned"] = result.get("products_mentioned", [])
+            state["sabian_mention_context"] = result.get("sabian_mention_context")
+            state["author_role"] = result.get("author_role", "unknown")
+            state["competitors_mentioned"] = result.get("competitors_mentioned", [])
+            state["thread_context_summary"] = result.get("thread_context_summary", "")
+
+            if not result.get("success", False) and result.get("error"):
+                state["processing_errors"] = state.get("processing_errors", []) + [result["error"]]
+
+            logger.debug(f"Extraction complete: is_relevant={state['is_relevant']}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Extraction node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["is_relevant"] = False
+            state["relevance_confidence"] = "low"
+            return state
+
+    def _analysis_node(self, state: BrandAnalysisState) -> BrandAnalysisState:
+        """
+        Analysis node: Deep sentiment and intent analysis for relevant posts.
+        """
+        try:
+            input_data = {
+                "cleaned_content": state.get("cleaned_content", ""),
+                "is_relevant": state.get("is_relevant", True),
+                "is_english": state.get("is_english", True),
+                "detected_language": state.get("detected_language", "English"),
+                "products_mentioned": state.get("products_mentioned", []),
+                "sabian_mention_context": state.get("sabian_mention_context"),
+                "author_role": state.get("author_role", "unknown"),
+                "competitors_mentioned": state.get("competitors_mentioned", []),
+                "thread_context_summary": state.get("thread_context_summary", "")
+            }
+
+            result = self.sentiment_analyzer.process(input_data)
+
+            if result.get("success", False):
+                # Sentiment
+                state["sentiment_level"] = result.get("sentiment_level")
+                state["emotion_type"] = result.get("emotion_type")
+                state["sentiment_confidence"] = result.get("sentiment_confidence", "medium")
+                state["sarcasm_detected"] = result.get("sarcasm_detected", False)
+
+                # Products
+                state["product_attributes"] = result.get("product_attributes", [])
+
+                # Competitive
+                state["competitor_products_owned"] = result.get("competitor_products_owned", [])
+                state["comparison_type"] = result.get("comparison_type")
+
+                # Journey
+                state["intents"] = result.get("intents", [])
+                state["purchase_stage"] = result.get("purchase_stage")
+                state["decision_drivers"] = result.get("decision_drivers", [])
+                state["pain_points"] = result.get("pain_points", [])
+                state["delight_factors"] = result.get("delight_factors", [])
+
+                # Notes
+                state["analysis_notes"] = result.get("analysis_notes", "")
+                state["analysis_skipped"] = result.get("analysis_skipped", False)
+                state["analysis_skip_reason"] = result.get("analysis_skip_reason", "")
+            else:
+                error_msg = f"Analysis failed: {result.get('error', 'Unknown error')}"
+                state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+
+            logger.debug(f"Analysis complete for post {state.get('post_id')}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Analysis node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            return state
+
+    def _validation_node(self, state: BrandAnalysisState) -> BrandAnalysisState:
+        """
+        Validation node: Rule-based validation and anomaly detection.
+        """
+        try:
+            result = self.output_validator.process(dict(state))
+
+            state["validation_passed"] = result.get("validation_passed", True)
+            state["validation_errors"] = result.get("validation_errors", [])
+            state["validation_warnings"] = result.get("validation_warnings", [])
+            state["validation_flags"] = result.get("validation_flags", [])
+            state["processing_status"] = result.get("processing_status", "completed")
+
+            # Set overall success
+            has_errors = len(state.get("processing_errors", [])) > 0
+            state["success"] = not has_errors or state.get("is_relevant") is not None
+
+            logger.debug(f"Validation complete: status={state['processing_status']}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Validation node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["validation_passed"] = False
+            state["processing_status"] = "validation_failed"
+            state["success"] = False
+            return state
+
+    def _route_after_preprocessing(self, state: BrandAnalysisState) -> str:
+        """
+        Determine routing after preprocessing.
+        """
+        # If empty content, skip to validation
+        if state.get("is_empty", False):
+            state["is_relevant"] = False
+            state["relevance_reason"] = "Empty content"
+            return "skip_to_validation"
+
+        # If not English, skip to validation
+        if not state.get("is_english", True):
+            state["is_relevant"] = False
+            state["relevance_reason"] = f"Non-English: {state.get('detected_language')}"
+            return "skip_to_validation"
+
+        # If no keywords found and no need for validation, skip
+        if (not state.get("preliminary_relevant", False) and
+            not state.get("needs_relevance_validation", False)):
+            state["is_relevant"] = False
+            state["relevance_reason"] = "No relevant keywords found"
+            return "skip_to_validation"
+
+        # Otherwise, go to extraction
+        return "extract"
+
+    def _route_after_extraction(self, state: BrandAnalysisState) -> str:
+        """
+        Determine routing after extraction.
+        """
+        if state.get("is_relevant", False):
+            return "analyze"
+        return "skip_to_validation"
+
+    def process_post(self, post_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a single forum post through the workflow.
+
+        Args:
+            post_data: Dictionary containing post data
+
+        Returns:
+            Dictionary with processed results
+        """
+        try:
+            initial_state = {
+                "post_id": post_data.get("post_id"),
+                "thread_id": post_data.get("thread_id"),
+                "post_author_id": post_data.get("post_author_id"),
+                "post_content": post_data.get("post_content", ""),
+                "thread_title": post_data.get("thread_title"),
+                "thread_first_post": post_data.get("thread_first_post"),
+                "thread_started_at": post_data.get("thread_started_at"),
+                "category_title": post_data.get("category_title"),
+                "category_topic": post_data.get("category_topic"),
+                "post_created_at": post_data.get("post_created_at"),
+                "processing_errors": [],
+                "success": True
+            }
+
+            final_state = self.workflow.invoke(initial_state)
+
+            return dict(final_state)
+
+        except Exception as e:
+            logger.error(f"Workflow execution error: {str(e)}")
+            return {
+                **post_data,
+                "success": False,
+                "processing_errors": [str(e)],
+                "processing_status": "workflow_error"
+            }
+
+    def process_batch(self, posts: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Process a batch of forum posts.
+
+        Args:
+            posts: List of post dictionaries
+
+        Returns:
+            List of processed post dictionaries
+        """
+        results = []
+        total = len(posts)
+
+        for idx, post in enumerate(posts, 1):
+            logger.info(f"Processing post {idx}/{total} (ID: {post.get('post_id')})")
+            result = self.process_post(post)
+            results.append(result)
+
+        logger.info(f"Batch processing complete: {total} posts processed")
+        return results

src/streamlit_app.py

@@ -1,40 +0,0 @@
-import altair as alt
-import numpy as np
-import pandas as pd
-import streamlit as st
-
-"""
-# Welcome to Streamlit!
-
-Edit `/streamlit_app.py` to customize this app to your heart's desire :heart:.
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
-
-In the meantime, below is an example of what you can do with just a few lines of code:
-"""
-
-num_points = st.slider("Number of points in spiral", 1, 10000, 1100)
-num_turns = st.slider("Number of turns in spiral", 1, 300, 31)
-
-indices = np.linspace(0, 1, num_points)
-theta = 2 * np.pi * num_turns * indices
-radius = indices
-
-x = radius * np.cos(theta)
-y = radius * np.sin(theta)
-
-df = pd.DataFrame({
-    "x": x,
-    "y": y,
-    "idx": indices,
-    "rand": np.random.randn(num_points),
-})
-
-st.altair_chart(alt.Chart(df, height=700, width=700)
-    .mark_point(filled=True)
-    .encode(
-        x=alt.X("x", axis=None),
-        y=alt.Y("y", axis=None),
-        color=alt.Color("idx", legend=None, scale=alt.Scale()),
-        size=alt.Size("rand", legend=None, scale=alt.Scale(range=[1, 150])),
-    ))

visualization/README.md

@@ -0,0 +1,347 @@
+# Musora Sentiment Analysis Dashboard
+
+A comprehensive, interactive Streamlit dashboard for visualizing sentiment analysis results from **multiple data sources**: social media comments (Facebook, Instagram, YouTube, Twitter) and Musora internal app comments across Musora brands (Drumeo, Pianote, Guitareo, Singeo).
+
+## Features
+
+### Main Dashboard
+- **Overall sentiment distribution** with interactive pie charts and gauge indicators
+- **Sentiment analysis by brand** (Drumeo, Pianote, Musora) with stacked bar charts
+- **Sentiment analysis by platform** (Facebook, Instagram, etc.) with percentage distributions
+- **Intent analysis** showing multi-label intent distributions (praise, question, request, etc.)
+- **Cross-dimensional heatmaps** showing negative sentiment by brand and platform
+- **Reply requirements analysis** with urgency breakdown
+- **Language distribution** analysis
+- **Temporal trends** with customizable time granularity (daily, weekly, monthly)
+- **Hierarchical sunburst** visualization for brand > platform > sentiment
+
+### Sentiment Analysis Page
+- **Multi-sentiment filtering** - Filter by any combination of sentiments (positive, negative, neutral, etc.) to analyze both good and bad performance
+- **Intent filtering** - Filter contents by specific user intents (question, praise, feedback_negative, etc.)
+- **Dynamic severity scoring** - Ranks contents based on selected sentiments, adapts calculations to your filter choices
+- **Advanced ranking controls** - Customize with minimum comment thresholds and multiple dynamic sort options
+- **Sort options** - Severity Score (balanced), Sentiment %, Sentiment Count (absolute), or Total Comments (volume)
+- **Engagement scatter plot** showing relationship between comment volume and sentiment
+- **Thumbnail display** for Musora internal app content (visual content previews)
+- **Detailed content analysis** with sentiment and intent distributions for each content
+- **AI-Powered Analysis** - Optional AI-generated insights and recommendations for each content
+- **View filtered comments** for each content with expandable sections
+- **Actionable insights** and recommendations based on sentiment patterns
+- **Export functionality** to download results as CSV with dynamic columns
+
+### Reply Required Page
+- **Prioritized comment queue** with urgency indicators (Urgent, High, Medium, Low)
+- **Smart filtering** by priority, platform, brand, and intent
+- **Pagination** for easy navigation through large comment lists
+- **Comment cards** showing full context (author, timestamp, sentiment, intent)
+- **Original and translated text** with expandable view for non-English comments
+- **Reply requirements by content** showing which contents need most attention
+- **Export functionality** for team collaboration or CRM import
+
+## Architecture
+
+```
+visualization/
+├── app.py                          # Main Streamlit application
+├── config/
+│   └── viz_config.json            # Configuration for colors, settings, queries
+├── data/
+│   └── data_loader.py             # Snowflake data loading with caching
+├── utils/
+│   ├── data_processor.py          # Data aggregation and processing
+│   └── metrics.py                 # Metrics calculation (KPIs, scores)
+├── components/
+│   ├── dashboard.py               # Main dashboard page
+│   ├── sentiment_analysis.py      # Comprehensive sentiment analysis page
+│   └── reply_required.py          # Reply management page
+├── visualizations/
+│   ├── sentiment_charts.py        # Sentiment visualization functions
+│   ├── distribution_charts.py     # Distribution visualization functions
+│   └── content_cards.py           # Display components and cards
+├── requirements.txt               # Python dependencies
+└── README.md                      # This file
+```
+
+## Installation
+
+### Prerequisites
+- Python 3.8+
+- Snowflake account with access to sentiment analysis data
+- Required environment variables in parent `.env` file:
+  - `SNOWFLAKE_USER`
+  - `SNOWFLAKE_PASSWORD`
+  - `SNOWFLAKE_ACCOUNT`
+  - `SNOWFLAKE_ROLE`
+  - `SNOWFLAKE_DATABASE`
+  - `SNOWFLAKE_WAREHOUSE`
+  - `SNOWFLAKE_SCHEMA`
+
+### Setup
+
+1. Navigate to the visualization directory:
+```bash
+cd visualization
+```
+
+2. Install dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+3. Ensure parent `.env` file is properly configured with Snowflake credentials
+
+## Usage
+
+### Running the Dashboard
+
+From the `visualization` directory:
+
+```bash
+streamlit run app.py
+```
+
+The dashboard will open in your default browser at `http://localhost:8501`
+
+### Navigation
+
+Use the sidebar to:
+- **Select pages** (Dashboard, Sentiment Analysis, Reply Required)
+- **Apply global filters** by platform, brand, sentiment, and date range
+- **Reload data** to fetch latest updates from Snowflake
+- **View data information** (record count, last update time)
+
+### Filtering Data
+
+1. Select desired filters in the sidebar:
+   - **Platforms**: Filter by data source (Facebook, Instagram, YouTube, Twitter, musora_app)
+   - **Brands**: Filter by Musora brand (Drumeo, Pianote, Guitareo, Singeo, Musora)
+   - **Sentiments**: Filter by sentiment polarity
+   - **Date Range**: Filter by comment timestamp
+
+2. Click "Apply Filters" to update visualizations
+
+3. Click "Reset Filters" to clear all filters
+
+### Exporting Data
+
+Each page provides export functionality:
+- **Sentiment Analysis**: Download top N contents as CSV with dynamic columns based on active filters
+- **Reply Required**: Download filtered comments as CSV
+
+## Configuration
+
+### Color Schemes
+
+Edit `config/viz_config.json` to customize:
+- **Sentiment colors**: Colors for each sentiment polarity
+- **Intent colors**: Colors for each intent category
+- **Platform colors**: Brand colors for each platform
+- **Brand colors**: Colors for each Musora brand
+
+### Dashboard Settings
+
+Configure in `viz_config.json`:
+- `default_date_range_days`: Default date range for filtering
+- `max_comments_display`: Maximum comments to display per page
+- `chart_height`: Default height for charts
+- `top_n_contents`: Number of contents to show in poor sentiment page
+
+### Data Query
+
+The Snowflake query is configured in `viz_config.json`:
+```json
+"snowflake": {
+    "query": "SELECT s.*, c.CHANNEL_NAME as BRAND, c.MESSAGE as CONTENT_DESCRIPTION, c.PERMALINK_URL FROM SOCIAL_MEDIA_DB.ML_FEATURES.COMMENT_SENTIMENT_FEATURES s JOIN SOCIAL_MEDIA_DB.CORE.DIM_CONTENT c ON s.CONTENT_SK = c.CONTENT_SK"
+}
+```
+
+## Extending the Dashboard
+
+### Adding New Pages
+
+1. Create a new component file in `components/`:
+```python
+# components/new_page.py
+def render_new_page(df):
+    st.title("New Page")
+    # Your page logic here
+```
+
+2. Import and add to navigation in `app.py`:
+```python
+from components.new_page import render_new_page
+
+# Add to page selection
+page = st.radio("Select Page", [..., "New Page"])
+
+# Add to page rendering
+elif page == "New Page":
+    render_new_page(df)
+```
+
+### Adding New Visualizations
+
+1. Add visualization function to appropriate module:
+   - `visualizations/sentiment_charts.py` for sentiment-related charts
+   - `visualizations/distribution_charts.py` for distribution charts
+
+2. Use the function in page components
+
+Example:
+```python
+def create_new_chart(df, title="New Chart"):
+    fig = go.Figure(...)
+    return fig
+```
+
+### Adding New Metrics
+
+Add calculation methods to `utils/metrics.py`:
+```python
+@staticmethod
+def calculate_new_metric(df):
+    # Your metric calculation
+    return metric_value
+```
+
+### Customizing Card Displays
+
+Modify display methods in `visualizations/content_cards.py`:
+```python
+@staticmethod
+def display_custom_card(data):
+    # Your custom card layout
+    pass
+```
+
+## Data Schema
+
+The dashboard expects the following columns from Snowflake:
+
+### Required Columns
+- `comment_sk`: Unique comment identifier
+- `comment_id`: Comment ID
+- `original_text`: Original comment text
+- `platform`: Social media platform
+- `brand`: Musora brand name
+- `sentiment_polarity`: Sentiment classification (very_positive, positive, neutral, negative, very_negative)
+- `intent`: Comma-separated intent labels
+- `requires_reply`: Boolean indicating if reply is needed
+- `content_sk`: Content identifier
+- `content_description`: Description of the content
+- `permalink_url`: URL to the original content
+
+### Optional Columns
+- `comment_timestamp`: When comment was posted
+- `processed_at`: When sentiment analysis was performed
+- `translated_text`: English translation for non-English comments
+- `detected_language`: Detected language of comment
+- `is_english`: Boolean indicating if comment is in English
+- `sentiment_confidence`: Confidence level of sentiment analysis
+- `author_name`: Comment author name
+- `channel_name`: Channel name
+- `thumbnail_url`: Content thumbnail URL (for Musora internal app content)
+- `parent_comment_id`: ID of parent comment (for replies)
+- `parent_comment_text`: Text of parent comment (for reply context)
+
+## Performance Optimization
+
+### Caching
+- Data loading is cached for 5 minutes using `@st.cache_data`
+- Clear cache using "Reload Data" button in sidebar
+
+### Pagination
+- Comments requiring reply are paginated (10 per page)
+- Reduces memory usage and improves rendering speed
+
+### Filtering
+- Apply filters to reduce dataset size before visualization
+- Filters are applied efficiently using pandas operations
+
+## Troubleshooting
+
+### Connection Issues
+- Verify Snowflake credentials in parent `.env` file
+- Check network connectivity to Snowflake
+- Ensure correct database, schema, and table names
+
+### No Data Displayed
+- Check if Snowflake query returns data
+- Verify column names match expected schema
+- Check applied filters - try resetting them
+
+### Slow Performance
+- Reduce date range in filters
+- Use "Apply Filters" to work with smaller datasets
+- Consider adding database indexes on frequently filtered columns
+
+### Visualization Errors
+- Check for missing or null values in data
+- Verify data types match expected types (dates, booleans, etc.)
+- Review browser console for JavaScript errors
+
+## Best Practices
+
+1. **Regular Data Updates**: Reload data periodically to see latest comments
+2. **Use Filters**: Apply filters to focus on specific segments
+3. **Export Insights**: Download CSV reports for offline analysis
+4. **Monitor Reply Queue**: Check "Reply Required" page daily
+5. **Track Trends**: Use temporal visualizations to identify patterns
+6. **Prioritize Urgent**: Address urgent replies (negative sentiment) first
+
+## Support
+
+For issues or feature requests:
+1. Check the troubleshooting section
+2. Review configuration files for correct settings
+3. Consult the main project README for sentiment analysis pipeline details
+
+## Version History
+
+### v1.3 (Current)
+- **Comprehensive Sentiment Analysis Page Redesign**
+  - Renamed "Poor Sentiment Contents" to "Sentiment Analysis" page
+  - **NEW: Multi-Sentiment Filtering** - Filter by any combination of sentiments (positive, negative, neutral, very_positive, very_negative)
+  - **NEW: Intent Filtering** - Filter contents by specific user intents (question, praise, feedback_negative, request, etc.)
+  - **Filter Status Indicator** - Visual feedback showing when filters are active
+
+- **Dynamic Ranking & Calculations**
+  - **Dynamic severity scoring** - Automatically calculates based on selected sentiments (not just negative)
+  - **Dynamic metrics** - Sentiment percentages and counts adapt to your filter selection
+  - Sort options now work with any sentiment combination
+
+- **Enhanced User Experience**
+  - Summary statistics that dynamically adapt to filters
+  - Contextual explanations that change based on selected sentiments
+  - Export with dynamic columns based on active filters
+  - Backward compatible - works like original when no filters selected
+
+- **New Use Cases Enabled**
+  - Analyze high-performing content (filter by positive sentiments)
+  - Identify successful patterns (combine sentiment + intent filters)
+  - Compare sentiment types side-by-side
+  - Focus on specific user behaviors
+
+### v1.2
+- **Multi-source data support** - Integrated Musora internal app comments alongside social media
+- **Smart severity scoring** - Content ranking now balances sentiment % with comment volume
+- **Advanced ranking controls** - Min comments filter and multiple sort options (severity, %, count, volume)
+- **Thumbnail display** - Visual content previews for Musora internal app content
+- **Platform disambiguation** - Renamed internal platform to "musora_app" to differentiate from "musora" brand
+- **Improved chart stability** - Fixed duplicate chart ID errors with unique keys
+- **Enhanced data schema** - Added support for thumbnail_url and parent comment fields
+
+### v1.1
+- **AI-Powered Agents** - ContentSummaryAgent for intelligent comment analysis
+- AI Analysis button on Sentiment Analysis page
+- LLM Helper with OpenAI API integration
+- Modular agent architecture ready for expansion
+
+### v1.0
+- Initial release
+- Main dashboard with comprehensive visualizations
+- Sentiment contents analysis page
+- Reply required management page
+- Global filtering and export functionality
+- Plotly-based interactive visualizations
+- Modular, extensible architecture

visualization/SnowFlakeConnection.py

@@ -0,0 +1,150 @@
+"""
+This class create a connection to Snowflake, run queries (read and write)
+"""
+import json
+import os
+from snowflake.snowpark import Session
+from dotenv import load_dotenv
+import logging
+logger = logging.getLogger()
+load_dotenv()
+
+class SnowFlakeConn:
+    def __init__(self):
+        self.session = self.connect_to_snowflake()
+
+
+    # =========================================================
+    def connect_to_snowflake(self):
+        # --- Snowflake connection via env vars ---
+        # Validate all required credentials exist
+        required_credentials = [
+            "SNOWFLAKE_USER",
+            "SNOWFLAKE_PASSWORD",
+            "SNOWFLAKE_ACCOUNT",
+            "SNOWFLAKE_ROLE",
+            "SNOWFLAKE_DATABASE",
+            "SNOWFLAKE_WAREHOUSE",
+            "SNOWFLAKE_SCHEMA"
+        ]
+
+        missing_credentials = []
+        for cred in required_credentials:
+            if not self.get_credential(cred):
+                missing_credentials.append(cred)
+
+        if missing_credentials:
+            error_msg = f"Missing required Snowflake credentials: {', '.join(missing_credentials)}"
+            logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        conn = dict(
+            user=self.get_credential("SNOWFLAKE_USER"),
+            password=self.get_credential("SNOWFLAKE_PASSWORD"),
+            account=self.get_credential("SNOWFLAKE_ACCOUNT"),
+            role=self.get_credential("SNOWFLAKE_ROLE"),
+            database=self.get_credential("SNOWFLAKE_DATABASE"),
+            warehouse=self.get_credential("SNOWFLAKE_WAREHOUSE"),
+            schema=self.get_credential("SNOWFLAKE_SCHEMA"),
+        )
+
+        try:
+            session = Session.builder.configs(conn).create()
+            logger.info("Successfully connected to Snowflake")
+            return session
+        except Exception as e:
+            logger.error(f"Failed to connect to Snowflake: {e}")
+            raise
+
+    # =========================================================
+    def get_credential(self, key):
+        return os.getenv(key)
+
+    # =========================================================
+    def run_read_query(self, query, data):
+        """
+        Executes a SQL query on Snowflake that fetch the data
+        :return: Pandas dataframe containing the query results
+        """
+
+        # Connect to Snowflake
+        try:
+            dataframe = self.session.sql(query).to_pandas()
+            dataframe.columns = dataframe.columns.str.lower()
+            print(f"reading {data} table successfully")
+            return dataframe
+        except Exception as e:
+            error_msg = f"Error reading {data}: {e}"
+            print(error_msg)
+            logger.error(error_msg)
+            raise
+
+    # =========================================================
+    def store_df_to_snowflake(self, table_name, dataframe, database="SOCIAL_MEDIA_DB", schema="ML_FEATURES", overwrite=False):
+        """
+        Executes a SQL query on Snowflake that write the preprocessed data on new tables
+        :param query: SQL query string to be executed
+        :return: None
+        """
+
+        try:
+            self.session.use_database(database)
+            self.session.use_schema(schema)
+
+            dataframe = dataframe.reset_index(drop=True)
+            dataframe.columns = dataframe.columns.str.upper()
+
+            self.session.write_pandas(df=dataframe,
+                                      table_name=table_name.strip().upper(),
+                                      auto_create_table=True,
+                                      overwrite=overwrite,
+                                      use_logical_type=True)
+            print(f"Data inserted into {table_name} successfully.")
+
+        except Exception as e:
+            print(f"Error in creating/updating/inserting table: {e}")
+
+    # =========================================================
+    def execute_sql_file(self, file_path):
+        """
+        Executes SQL queries from a file
+        :param file_path: Path to SQL file
+        :return: Query result or None for DDL/DML
+        """
+        try:
+            with open(file_path, 'r', encoding='utf-8') as file:
+                sql_content = file.read()
+
+            result = self.session.sql(sql_content).collect()
+            print(f"Successfully executed SQL from {file_path}")
+            return result
+        except Exception as e:
+            print(f"Error executing SQL file {file_path}: {e}")
+            return None
+
+    # =========================================================
+    def execute_query(self, query, description="query"):
+        """
+        Executes a SQL query and returns results
+        :param query: SQL query string
+        :param description: Description of the query for logging
+        :return: Query results
+        """
+        try:
+            result = self.session.sql(query).collect()
+            print(f"Successfully executed {description}")
+            return result
+        except Exception as e:
+            print(f"Error executing {description}: {e}")
+            return None
+
+
+    # =========================================================
+    def get_data(self, data):
+        # get any sort of data based on requirement --> comments, contents, etc
+        pass
+
+    # =========================================================
+    def close_connection(self):
+        self.session.close()
+

visualization/agents/README.md

@@ -0,0 +1,320 @@
+# Visualization Agents
+
+## Overview
+This folder contains AI-powered agents that enhance the sentiment analysis dashboard with intelligent, context-aware insights and analysis capabilities.
+
+## Architecture
+
+### Base Agent Pattern
+All agents inherit from `BaseVisualizationAgent` which provides:
+- Common interface (`process()`, `validate_input()`)
+- Error handling
+- Logging functionality
+- Consistent configuration
+
+### LLM Helper
+`utils/llm_helper.py` provides:
+- OpenAI API integration
+- Retry logic with exponential backoff
+- JSON mode support
+- Token usage tracking
+
+## Available Agents
+
+### 1. ContentSummaryAgent
+
+**Purpose**: Analyze and summarize comments for content pieces
+
+**Location**: `agents/content_summary_agent.py`
+
+**Input**:
+```python
+{
+    'content_sk': str,              # Content identifier
+    'content_description': str,      # Content title/description
+    'comments': DataFrame or list    # Comments data
+}
+```
+
+**Output**:
+```python
+{
+    'success': bool,
+    'content_sk': str,
+    'summary': {
+        'executive_summary': str,              # 2-3 sentence overview
+        'main_themes': [                       # Top themes discussed
+            {
+                'theme': str,
+                'sentiment': str,  # positive/negative/mixed
+                'description': str
+            }
+        ],
+        'praise_points': [str],                # What users love
+        'key_complaints': [str],               # Main concerns
+        'frequently_asked_questions': [str],   # Common questions
+        'unexpected_insights': [str],          # Surprising patterns
+        'action_recommendations': [            # Suggested actions
+            {
+                'priority': str,   # high/medium/low
+                'action': str
+            }
+        ]
+    },
+    'metadata': {
+        'total_comments_analyzed': int,
+        'model_used': str,
+        'tokens_used': int
+    }
+}
+```
+
+**Configuration**:
+- Model: `gpt-5-nano` (configurable)
+- Temperature: 0.3 (lower for focused summaries)
+- Sampling: All negative comments + up to 50 positive/neutral (if >100 total)
+
+**Features**:
+- **Smart sampling**: Prioritizes negative comments, samples others
+- **Context preservation**: Includes sentiment and intent metadata
+- **Token optimization**: Truncates long comments to 300 chars
+- **Structured output**: JSON format with guaranteed fields
+- **Error handling**: Graceful failures with retry capability
+
+## UI Integration
+
+### Poor Sentiment Contents Page
+
+**Location**: `components/poor_sentiment_contents.py`
+
+**User Flow**:
+1. User views content cards on Poor Sentiment Contents page
+2. Clicks "🔍 Generate AI Analysis" button
+3. Agent processes comments (with spinner indicator)
+4. Summary displays in expandable section
+5. Result cached in session state
+
+**Display Sections**:
+- **Executive Summary**: High-level overview (info box)
+- **Main Themes**: Key topics with sentiment indicators
+- **Praise Points** ✅ & **Key Complaints** ⚠️ (side-by-side)
+- **FAQs** ❓ & **Unexpected Insights** 💡 (side-by-side)
+- **Recommended Actions** 🎯 (priority-coded)
+- **Analysis Metadata** ℹ️ (expandable details)
+
+**Session Caching**:
+- Summaries stored in `st.session_state.content_summaries`
+- Key: `content_sk`
+- Persists during session, cleared on page reload
+- Prevents redundant API calls
+
+## Usage Example
+
+```python
+from agents.content_summary_agent import ContentSummaryAgent
+import pandas as pd
+
+# Initialize agent
+agent = ContentSummaryAgent(model="gpt-5-nano", temperature=0.3)
+
+# Prepare input
+input_data = {
+    'content_sk': '12345',
+    'content_description': 'Advanced Drum Fills Tutorial',
+    'comments': comments_df  # DataFrame with comments
+}
+
+# Generate summary
+result = agent.process(input_data)
+
+if result['success']:
+    summary = result['summary']
+    print(summary['executive_summary'])
+
+    for theme in summary['main_themes']:
+        print(f"Theme: {theme['theme']} ({theme['sentiment']})")
+        print(f"  {theme['description']}")
+else:
+    print(f"Error: {result['error']}")
+```
+
+## Environment Setup
+
+### Required Environment Variables
+Add to `.env` file (parent directory):
+```bash
+OPENAI_API_KEY=your_openai_api_key_here
+```
+
+### Dependencies
+All dependencies already in `visualization/requirements.txt`:
+- `streamlit>=1.28.0`
+- `pandas>=2.0.0`
+- `python-dotenv>=1.0.0`
+- OpenAI library (inherited from parent project)
+
+## Error Handling
+
+### Agent-Level Errors
+- **Invalid input**: Returns `{'success': False, 'error': 'Invalid input data'}`
+- **LLM API failure**: Retries up to 3 times with exponential backoff
+- **JSON parsing error**: Returns error with raw content
+- **Exception**: Catches all exceptions, logs, returns error dict
+
+### UI-Level Errors
+- Displays error message in red box
+- Provides "🔄 Retry Analysis" button
+- Clears cache and regenerates on retry
+- Logs errors to agent logger
+
+## Performance Considerations
+
+### API Costs
+- Model: `gpt-5-nano` (cost-effective)
+- Sampling strategy: Reduces tokens by up to 50% for large comment sets
+- Comment truncation: Max 300 chars per comment
+- Session caching: Eliminates duplicate API calls
+
+### Response Time
+- Average: 5-10 seconds for 50-100 comments
+- Depends on: Comment count, OpenAI API latency
+- User feedback: Spinner shows "Analyzing comments with AI..."
+
+### Scalability
+- Handles up to 100 comments per analysis (after sampling)
+- Parallel requests: Each content analyzed independently
+- Session state: Memory usage scales with number of analyzed contents
+
+## Extending Agents
+
+### Adding New Agents
+
+1. **Create agent file**:
+```python
+# agents/new_agent.py
+from agents.base_agent import BaseVisualizationAgent
+from utils.llm_helper import LLMHelper
+
+class NewAgent(BaseVisualizationAgent):
+    def __init__(self, model="gpt-5-nano", temperature=0.7):
+        super().__init__(name="NewAgent", model=model, temperature=temperature)
+        self.llm_helper = LLMHelper(model=model, temperature=temperature)
+
+    def validate_input(self, input_data):
+        # Validation logic
+        return True
+
+    def process(self, input_data):
+        # Processing logic
+        pass
+```
+
+2. **Update `__init__.py`**:
+```python
+from .new_agent import NewAgent
+
+__all__ = ['ContentSummaryAgent', 'NewAgent']
+```
+
+3. **Integrate in UI**:
+- Import agent in component file
+- Add UI controls (buttons, inputs)
+- Display results
+- Handle caching if needed
+
+### Best Practices
+
+1. **Input Validation**: Always validate required fields
+2. **Error Handling**: Use `handle_error()` method
+3. **Logging**: Use `log_processing()` for debugging
+4. **Structured Output**: Return consistent dict format
+5. **Caching**: Use session state for expensive operations
+6. **Token Optimization**: Sample/truncate data for large inputs
+7. **User Feedback**: Show spinners for async operations
+8. **Graceful Degradation**: Provide fallbacks for failures
+
+## Testing
+
+### Manual Testing
+1. Start dashboard: `streamlit run app.py`
+2. Navigate to "⚠️ Poor Sentiment Contents" page
+3. Click "🔍 Generate AI Analysis" for any content
+4. Verify summary displays correctly
+5. Check session caching (click button again)
+6. Test error handling (disconnect network)
+
+### Unit Testing
+```python
+# tests/test_content_summary_agent.py
+import pytest
+from agents.content_summary_agent import ContentSummaryAgent
+
+def test_validate_input():
+    agent = ContentSummaryAgent()
+
+    # Valid input
+    valid_input = {
+        'content_sk': '123',
+        'content_description': 'Test',
+        'comments': []
+    }
+    assert agent.validate_input(valid_input) == True
+
+    # Missing field
+    invalid_input = {'content_sk': '123'}
+    assert agent.validate_input(invalid_input) == False
+```
+
+## Future Enhancements
+
+### Planned Features
+1. **Batch Analysis**: Analyze multiple contents at once
+2. **Trend Detection**: Compare with historical summaries
+3. **Export Summaries**: Download as PDF/CSV
+4. **Custom Prompts**: User-defined analysis focus
+5. **Multi-language Support**: Summaries in user's language
+
+### Additional Agents (Roadmap)
+- **InsightsSummaryAgent**: Overall dataset insights
+- **InteractiveChatbotAgent**: Conversational analysis
+- **ComparativeContentAgent**: Content comparison
+- **ReplySuggestionAgent**: Generate reply suggestions
+- **TrendForecastingAgent**: Predict sentiment trends
+
+## Troubleshooting
+
+### Common Issues
+
+**Issue**: `OPENAI_API_KEY not found`
+- **Solution**: Add key to `.env` file in parent directory
+
+**Issue**: Import error for `agents` module
+- **Solution**: Ensure `__init__.py` exists in `visualization/agents/`
+
+**Issue**: LLM timeout errors
+- **Solution**: Reduce comment count or increase retry limit
+
+**Issue**: JSON parsing errors
+- **Solution**: Check LLM prompt format, ensure JSON mode enabled
+
+**Issue**: Cached summaries not showing
+- **Solution**: Check `st.session_state.content_summaries` initialization
+
+## Support
+
+For issues or questions:
+1. Check this README
+2. Review agent logs in console
+3. Inspect session state in Streamlit
+4. Verify environment variables
+5. Check OpenAI API status
+
+## Version History
+
+### v1.0.0 (Current)
+- Initial release
+- ContentSummaryAgent implementation
+- Poor Sentiment Contents page integration
+- Session-based caching
+- Error handling and retry logic
+- Comprehensive UI display

visualization/agents/__init__.py

@@ -0,0 +1,8 @@
+"""
+Visualization Agents Package
+Contains AI agents for intelligent dashboard features
+"""
+
+from .content_summary_agent import ContentSummaryAgent
+
+__all__ = ['ContentSummaryAgent']

visualization/agents/base_agent.py

@@ -0,0 +1,88 @@
+"""
+Base Agent class for visualization agents
+Provides common functionality and interface for all agents
+"""
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+import logging
+
+
+class BaseVisualizationAgent(ABC):
+    """
+    Abstract base class for all visualization agents
+    """
+
+    def __init__(self, name: str, model: str = "gpt-5-nano", temperature: float = 0.7):
+        """
+        Initialize base agent
+
+        Args:
+            name: Agent name
+            model: LLM model to use
+            temperature: LLM temperature
+        """
+        self.name = name
+        self.model = model
+        self.temperature = temperature
+        self.logger = logging.getLogger(f"visualization.agents.{name}")
+
+    @abstractmethod
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process input data and return results
+
+        Args:
+            input_data: Input data dictionary
+
+        Returns:
+            Results dictionary
+        """
+        pass
+
+    @abstractmethod
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate input data
+
+        Args:
+            input_data: Input data dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        pass
+
+    def log_processing(self, message: str, level: str = "info"):
+        """
+        Log processing information
+
+        Args:
+            message: Log message
+            level: Log level (info, warning, error)
+        """
+        log_func = getattr(self.logger, level.lower(), self.logger.info)
+        log_func(f"[{self.name}] {message}")
+
+    def handle_error(self, error: Exception, context: str = "") -> Dict[str, Any]:
+        """
+        Handle errors consistently
+
+        Args:
+            error: Exception that occurred
+            context: Additional context information
+
+        Returns:
+            Error response dictionary
+        """
+        error_msg = f"Error in {self.name}: {str(error)}"
+        if context:
+            error_msg += f" | Context: {context}"
+
+        self.log_processing(error_msg, level="error")
+
+        return {
+            'success': False,
+            'error': str(error),
+            'error_type': type(error).__name__,
+            'context': context
+        }

visualization/agents/content_summary_agent.py

@@ -0,0 +1,366 @@
+"""
+Content Summary Agent
+Analyzes and summarizes comments for content pieces
+"""
+import pandas as pd
+from typing import Dict, Any, List
+import sys
+from pathlib import Path
+
+# Add parent directory to path
+parent_dir = Path(__file__).resolve().parent.parent
+sys.path.append(str(parent_dir))
+
+from agents.base_agent import BaseVisualizationAgent
+from utils.llm_helper import LLMHelper
+
+
+class ContentSummaryAgent(BaseVisualizationAgent):
+    """
+    Agent that analyzes and summarizes comments for content
+    Extracts themes, praise points, complaints, FAQs, and insights
+    """
+
+    def __init__(self, model: str = "gpt-5-nano", temperature: float = 1):
+        """
+        Initialize Content Summary Agent
+
+        Args:
+            model: LLM model to use
+            temperature: Temperature for generation (lower for more focused summaries)
+        """
+        super().__init__(name="ContentSummaryAgent", model=model, temperature=temperature)
+        self.llm_helper = LLMHelper(model=model, temperature=temperature)
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate input data
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        required_fields = ['content_sk', 'content_description', 'comments']
+
+        for field in required_fields:
+            if field not in input_data:
+                self.log_processing(f"Missing required field: {field}", level="error")
+                return False
+
+        if not isinstance(input_data['comments'], (list, pd.DataFrame)):
+            self.log_processing("Comments must be a list or DataFrame", level="error")
+            return False
+
+        return True
+
+    def _prepare_comments_context(self, comments: Any, sentiment_type: str = 'negative') -> str:
+        """
+        Prepare comments data for LLM analysis
+
+        Args:
+            comments: Comments as DataFrame or list of dicts
+            sentiment_type: Type of sentiment to analyze ('negative', 'positive', 'combined')
+
+        Returns:
+            Formatted string with comment data
+        """
+        # Convert to DataFrame if needed
+        if isinstance(comments, list):
+            comments_df = pd.DataFrame(comments)
+        else:
+            comments_df = comments.copy()
+
+        # Filter based on sentiment type
+        if sentiment_type == 'negative':
+            # Only negative comments
+            comments_df = comments_df[
+                comments_df['sentiment_polarity'].isin(['negative', 'very_negative'])
+            ]
+        elif sentiment_type == 'positive':
+            # Only positive comments
+            comments_df = comments_df[
+                comments_df['sentiment_polarity'].isin(['positive', 'very_positive'])
+            ]
+        # else: combined - use all comments
+
+        # Limit to reasonable number for API
+        if len(comments_df) > 100:
+            if sentiment_type == 'combined':
+                # For combined: sample from both positive and negative
+                negative_comments = comments_df[
+                    comments_df['sentiment_polarity'].isin(['negative', 'very_negative'])
+                ].sample(n=min(50, len(comments_df[comments_df['sentiment_polarity'].isin(['negative', 'very_negative'])])), random_state=42)
+
+                positive_comments = comments_df[
+                    comments_df['sentiment_polarity'].isin(['positive', 'very_positive'])
+                ].sample(n=min(50, len(comments_df[comments_df['sentiment_polarity'].isin(['positive', 'very_positive'])])), random_state=42)
+
+                comments_df = pd.concat([negative_comments, positive_comments])
+            else:
+                # For single sentiment type: just sample
+                comments_df = comments_df.sample(n=min(100, len(comments_df)), random_state=42)
+
+        # Format comments for analysis
+        comments_text = []
+        for idx, row in comments_df.iterrows():
+            text = row.get('display_text', row.get('original_text', ''))
+            sentiment = row.get('sentiment_polarity', 'unknown')
+            intent = row.get('intent', 'unknown')
+
+            comment_entry = f"""
+Comment #{idx + 1}:
+- Text: {text[:300]}{'...' if len(str(text)) > 300 else ''}
+- Sentiment: {sentiment}
+- Intent: {intent}
+"""
+            comments_text.append(comment_entry)
+
+        return "\n".join(comments_text)
+
+    def _generate_summary_prompt(
+        self,
+        content_description: str,
+        comments_context: str,
+        total_comments: int,
+        sentiment_type: str = 'negative'
+    ) -> str:
+        """
+        Generate prompt for LLM
+
+        Args:
+            content_description: Description of the content
+            comments_context: Formatted comments
+            total_comments: Total number of comments
+            sentiment_type: Type of sentiment being analyzed ('negative', 'positive', 'combined')
+
+        Returns:
+            Prompt string
+        """
+        # Customize prompt based on sentiment type
+        if sentiment_type == 'negative':
+            focus_instruction = "Focus on understanding negative feedback, complaints, and issues that need attention."
+        elif sentiment_type == 'positive':
+            focus_instruction = "Focus on understanding what users love, praise points, and successful elements that should be maintained or amplified."
+        else:  # combined
+            focus_instruction = "Provide a balanced analysis covering both positive feedback and areas for improvement."
+
+        prompt = f"""Analyze the {sentiment_type} comments below for the following content and provide a brief executive summary.
+
+**Content:** {content_description}
+
+**Total Comments Analyzed:** {total_comments}
+
+**Analysis Focus:** {focus_instruction}
+
+**Comments to Analyze:**
+{comments_context}
+
+**Task:** Provide a concise executive summary in JSON format with the following structure:
+
+{{
+    "executive_summary": "2-3 sentence high-level overview focusing on {sentiment_type} sentiment",
+    "main_themes": [
+        {{
+            "theme": "theme name",
+            "sentiment": "positive/negative/mixed",
+            "description": "brief description"
+        }}
+    ],
+    "praise_points": ["point 1", "point 2", "point 3"],
+    "key_complaints": ["complaint 1", "complaint 2", "complaint 3"],
+    "frequently_asked_questions": ["question 1", "question 2"],
+    "unexpected_insights": ["insight 1", "insight 2"],
+    "action_recommendations": [
+        {{
+            "priority": "high/medium/low",
+            "action": "recommended action"
+        }}
+    ]
+}}
+
+**Guidelines:**
+- Be concise and actionable
+- Focus on the most important insights from {sentiment_type} comments
+- Limit each list to top 3-5 items
+- If a section has no relevant items, use an empty list
+- Executive summary should capture the overall patterns and key takeaways
+"""
+        return prompt
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process comments and generate summary
+
+        Args:
+            input_data: {
+                'content_sk': content identifier,
+                'content_description': content title/description,
+                'comments': DataFrame or list of comment dicts,
+                'sentiment_type': 'negative', 'positive', or 'combined' (optional, defaults to 'negative')
+            }
+
+        Returns:
+            {
+                'success': bool,
+                'content_sk': str,
+                'sentiment_type': str,
+                'summary': {
+                    'executive_summary': str,
+                    'main_themes': list,
+                    'praise_points': list,
+                    'key_complaints': list,
+                    'frequently_asked_questions': list,
+                    'unexpected_insights': list,
+                    'action_recommendations': list
+                },
+                'metadata': {
+                    'total_comments_analyzed': int,
+                    'model_used': str,
+                    'tokens_used': int
+                }
+            }
+        """
+        try:
+            # Validate input
+            if not self.validate_input(input_data):
+                return {
+                    'success': False,
+                    'error': 'Invalid input data',
+                    'content_sk': input_data.get('content_sk', 'unknown')
+                }
+
+            content_sk = input_data['content_sk']
+            content_description = input_data['content_description']
+            comments = input_data['comments']
+            sentiment_type = input_data.get('sentiment_type', 'negative')  # Default to negative for backward compatibility
+
+            self.log_processing(f"Starting {sentiment_type} analysis for content: {content_sk}")
+
+            # Convert to DataFrame if needed
+            if isinstance(comments, list):
+                comments_df = pd.DataFrame(comments)
+            else:
+                comments_df = comments.copy()
+
+            total_comments = len(comments_df)
+
+            if total_comments == 0:
+                return {
+                    'success': True,
+                    'content_sk': content_sk,
+                    'sentiment_type': sentiment_type,
+                    'summary': {
+                        'executive_summary': 'No comments available for analysis.',
+                        'main_themes': [],
+                        'praise_points': [],
+                        'key_complaints': [],
+                        'frequently_asked_questions': [],
+                        'unexpected_insights': [],
+                        'action_recommendations': []
+                    },
+                    'metadata': {
+                        'total_comments_analyzed': 0,
+                        'model_used': self.model,
+                        'tokens_used': 0
+                    }
+                }
+
+            # Prepare comments context based on sentiment type
+            comments_context = self._prepare_comments_context(comments_df, sentiment_type)
+
+            # Get count of comments after filtering
+            if sentiment_type == 'negative':
+                filtered_count = len(comments_df[comments_df['sentiment_polarity'].isin(['negative', 'very_negative'])])
+            elif sentiment_type == 'positive':
+                filtered_count = len(comments_df[comments_df['sentiment_polarity'].isin(['positive', 'very_positive'])])
+            else:
+                filtered_count = total_comments
+
+            if filtered_count == 0:
+                return {
+                    'success': True,
+                    'content_sk': content_sk,
+                    'sentiment_type': sentiment_type,
+                    'summary': {
+                        'executive_summary': f'No {sentiment_type} comments available for analysis.',
+                        'main_themes': [],
+                        'praise_points': [],
+                        'key_complaints': [],
+                        'frequently_asked_questions': [],
+                        'unexpected_insights': [],
+                        'action_recommendations': []
+                    },
+                    'metadata': {
+                        'total_comments_analyzed': 0,
+                        'model_used': self.model,
+                        'tokens_used': 0
+                    }
+                }
+
+            # Generate prompt
+            prompt = self._generate_summary_prompt(
+                content_description,
+                comments_context,
+                filtered_count,
+                sentiment_type
+            )
+
+            # System message
+            system_message = """You are an expert social media analyst specializing in
+sentiment analysis and community insights. Provide concise, actionable summaries
+that help content creators understand their audience feedback."""
+
+            # Get LLM response
+            self.log_processing(f"Calling LLM for {sentiment_type} summary generation")
+            response = self.llm_helper.get_structured_completion(
+                prompt=prompt,
+                system_message=system_message,
+                max_retries=3
+            )
+
+            if not response['success']:
+                return self.handle_error(
+                    Exception(response.get('error', 'LLM call failed')),
+                    context=f"content_sk={content_sk}, sentiment_type={sentiment_type}"
+                )
+
+            # Extract summary
+            summary = response['content']
+
+            # Ensure all expected fields exist
+            default_summary = {
+                'executive_summary': '',
+                'main_themes': [],
+                'praise_points': [],
+                'key_complaints': [],
+                'frequently_asked_questions': [],
+                'unexpected_insights': [],
+                'action_recommendations': []
+            }
+
+            # Merge with defaults
+            for key in default_summary:
+                if key not in summary:
+                    summary[key] = default_summary[key]
+
+            self.log_processing(f"Successfully generated {sentiment_type} summary for content: {content_sk}")
+
+            return {
+                'success': True,
+                'content_sk': content_sk,
+                'sentiment_type': sentiment_type,
+                'summary': summary,
+                'metadata': {
+                    'total_comments_analyzed': filtered_count,
+                    'model_used': response['model'],
+                    'tokens_used': response['usage']['total_tokens']
+                }
+            }
+
+        except Exception as e:
+            return self.handle_error(
+                e,
+                context=f"content_sk={input_data.get('content_sk', 'unknown')}, sentiment_type={input_data.get('sentiment_type', 'negative')}"
+            )

visualization/app.py

@@ -0,0 +1,180 @@
+"""
+Musora Sentiment Analysis Dashboard
+Main Streamlit Application
+
+Run with: streamlit run app.py
+"""
+import streamlit as st
+import sys
+from pathlib import Path
+import json
+
+# Add parent directory to path
+parent_dir = Path(__file__).resolve().parent
+sys.path.append(str(parent_dir))
+
+from data.data_loader import SentimentDataLoader
+from components.dashboard import render_dashboard
+from components.sentiment_analysis import render_sentiment_analysis
+from components.reply_required import render_reply_required
+
+
+# Load configuration
+config_path = parent_dir / "config" / "viz_config.json"
+with open(config_path, 'r') as f:
+    config = json.load(f)
+
+# Page configuration
+st.set_page_config(
+    page_title=config['page_config']['page_title'],
+    page_icon=config['page_config']['page_icon'],
+    layout=config['page_config']['layout'],
+    initial_sidebar_state=config['page_config']['initial_sidebar_state']
+)
+
+
+def main():
+    """
+    Main application function
+    """
+    # Sidebar
+    with st.sidebar:
+        st.image("visualization/img/musora.png", use_container_width=True)
+        st.title("Navigation")
+
+        # Page selection
+        page = st.radio(
+            "Select Page",
+            ["📊 Dashboard", "🔍 Sentiment Analysis", "💬 Reply Required"],
+            index=0
+        )
+
+        st.markdown("---")
+
+        # Filters section
+        st.markdown("### 🔍 Global Filters")
+
+        # Initialize session state for filters
+        if 'filters_applied' not in st.session_state:
+            st.session_state.filters_applied = False
+
+        # Load data first to get filter options
+        with st.spinner("Loading data..."):
+            data_loader = SentimentDataLoader()
+            df = data_loader.load_data()
+
+        if df.empty:
+            st.error("No data available. Please check your Snowflake connection.")
+            return
+
+        # Get filter options
+        filter_options = data_loader.get_filter_options(df)
+
+        # Platform filter
+        selected_platforms = st.multiselect(
+            "Platforms",
+            options=filter_options['platforms'],
+            default=[]
+        )
+
+        # Brand filter
+        selected_brands = st.multiselect(
+            "Brands",
+            options=filter_options['brands'],
+            default=[]
+        )
+
+        # Sentiment filter
+        selected_sentiments = st.multiselect(
+            "Sentiments",
+            options=filter_options['sentiments'],
+            default=[]
+        )
+
+        # Date range filter (if available)
+        if 'comment_timestamp' in df.columns and not df.empty:
+            min_date = df['comment_timestamp'].min().date()
+            max_date = df['comment_timestamp'].max().date()
+
+            date_range = st.date_input(
+                "Date Range",
+                value=(min_date, max_date),
+                min_value=min_date,
+                max_value=max_date
+            )
+        else:
+            date_range = None
+
+        # Apply filters button
+        if st.button("🔍 Apply Filters", use_container_width=True):
+            st.session_state.filters_applied = True
+
+        # Reset filters button
+        if st.button("🔄 Reset Filters", use_container_width=True):
+            st.session_state.filters_applied = False
+            st.rerun()
+
+        st.markdown("---")
+
+        # Data refresh
+        st.markdown("### 🔄 Data Management")
+
+        if st.button("♻️ Reload Data", use_container_width=True):
+            st.cache_data.clear()
+            st.rerun()
+
+        # Display data info
+        st.markdown("---")
+        st.markdown("### ℹ️ Data Info")
+        st.info(f"**Total Records:** {len(df):,}")
+
+        if 'processed_at' in df.columns and not df.empty:
+            last_update = df['processed_at'].max()
+            st.info(f"**Last Updated:** {last_update.strftime('%Y-%m-%d %H:%M')}")
+
+    # Apply filters if needed
+    if st.session_state.filters_applied:
+        df = data_loader.apply_filters(
+            df,
+            platforms=selected_platforms if selected_platforms else None,
+            brands=selected_brands if selected_brands else None,
+            sentiments=selected_sentiments if selected_sentiments else None,
+            date_range=date_range if date_range and len(date_range) == 2 else None
+        )
+
+        # Show filter summary
+        if df.empty:
+            st.warning("No data matches the selected filters. Please adjust your filters.")
+            return
+        else:
+            st.info(f"Showing {len(df):,} records after applying filters")
+
+    # Main content area - render selected page
+    if page == "📊 Dashboard":
+        render_dashboard(df)
+
+    elif page == "🔍 Sentiment Analysis":
+        render_sentiment_analysis(df)
+
+    elif page == "💬 Reply Required":
+        render_reply_required(df)
+
+    # Footer
+    st.markdown("---")
+    st.markdown(
+        """
+        <div style='text-align: center; color: gray; padding: 20px;'>
+            <p>Musora Sentiment Analysis Dashboard v1.0</p>
+            <p>Powered by Streamlit | Data from Snowflake</p>
+        </div>
+        """,
+        unsafe_allow_html=True
+    )
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        st.error(f"An error occurred: {str(e)}")
+        st.exception(e)

visualization/components/dashboard.py

@@ -0,0 +1,538 @@
+"""
+Main Dashboard Page
+Displays overall sentiment distributions by brand and platform
+"""
+import streamlit as st
+import sys
+from pathlib import Path
+
+# Add parent directory to path
+parent_dir = Path(__file__).resolve().parent.parent
+sys.path.append(str(parent_dir))
+
+from utils.data_processor import SentimentDataProcessor
+from utils.metrics import SentimentMetrics
+from visualizations.sentiment_charts import SentimentCharts
+from visualizations.distribution_charts import DistributionCharts
+from visualizations.demographic_charts import DemographicCharts
+from visualizations.content_cards import ContentCards
+
+
+def render_dashboard(df):
+    """
+    Render the main dashboard page
+
+    Args:
+        df: Sentiment dataframe
+    """
+    st.title("📊 Sentiment Analysis Dashboard")
+
+    # Performance tip
+    if len(df) > 10000:
+        st.info(f"💡 **Performance Tip**: Loaded {len(df):,} comments. Use the global filters in the sidebar to narrow down your analysis for faster performance.")
+
+    st.markdown("---")
+
+    # Initialize components
+    sentiment_charts = SentimentCharts()
+    distribution_charts = DistributionCharts()
+    processor = SentimentDataProcessor()
+
+    # Display overall summary statistics
+    ContentCards.display_summary_stats(df)
+
+    st.markdown("---")
+
+    # Calculate overall metrics
+    overall_metrics = SentimentMetrics.calculate_overall_metrics(df)
+
+    # Display health indicator
+    col1, col2, col3 = st.columns([1, 2, 1])
+    with col2:
+        ContentCards.display_health_indicator(overall_metrics['negative_pct'])
+
+    st.markdown("---")
+
+    # Overall sentiment distribution
+    st.markdown("## 🎯 Overall Sentiment Distribution")
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        # Sentiment pie chart
+        sentiment_pie = sentiment_charts.create_sentiment_pie_chart(df, title="Overall Sentiment Distribution")
+        st.plotly_chart(sentiment_pie, use_container_width=True)
+
+    with col2:
+        # Sentiment score gauge
+        sentiment_gauge = sentiment_charts.create_sentiment_score_gauge(
+            overall_metrics['avg_sentiment_score'],
+            title="Overall Sentiment Score"
+        )
+        st.plotly_chart(sentiment_gauge, use_container_width=True)
+
+        # Additional metrics
+        metric_col1, metric_col2 = st.columns(2)
+        with metric_col1:
+            st.metric("Positive %", f"{overall_metrics['positive_pct']:.1f}%")
+        with metric_col2:
+            st.metric("Reply Rate %", f"{overall_metrics['reply_required_pct']:.1f}%")
+
+    st.markdown("---")
+
+    # Sentiment by Brand
+    st.markdown("## 🏢 Sentiment Analysis by Brand")
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        # Stacked bar chart
+        brand_sentiment_bar = sentiment_charts.create_sentiment_bar_chart(
+            df, group_by='brand', title="Sentiment Distribution by Brand"
+        )
+        st.plotly_chart(brand_sentiment_bar, use_container_width=True)
+
+    with col2:
+        # Percentage bar chart
+        brand_sentiment_pct = sentiment_charts.create_sentiment_percentage_bar_chart(
+            df, group_by='brand', title="Sentiment Distribution by Brand (%)"
+        )
+        st.plotly_chart(brand_sentiment_pct, use_container_width=True)
+
+    # Brand metrics table
+    with st.expander("📈 Detailed Brand Metrics"):
+        brand_metrics = SentimentMetrics.calculate_brand_metrics(df)
+
+        brand_data = []
+        for brand, metrics in brand_metrics.items():
+            brand_data.append({
+                'Brand': brand.title(),
+                'Total Comments': metrics['total_comments'],
+                'Replies Needed': metrics['total_reply_required'],
+                'Negative %': f"{metrics['negative_pct']:.1f}%",
+                'Positive %': f"{metrics['positive_pct']:.1f}%",
+                'Avg Sentiment Score': f"{metrics['avg_sentiment_score']:.2f}"
+            })
+
+        st.table(brand_data)
+
+    st.markdown("---")
+
+    # Sentiment by Platform
+    st.markdown("## 🌐 Sentiment Analysis by Platform")
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        # Stacked bar chart
+        platform_sentiment_bar = sentiment_charts.create_sentiment_bar_chart(
+            df, group_by='platform', title="Sentiment Distribution by Platform"
+        )
+        st.plotly_chart(platform_sentiment_bar, use_container_width=True)
+
+    with col2:
+        # Percentage bar chart
+        platform_sentiment_pct = sentiment_charts.create_sentiment_percentage_bar_chart(
+            df, group_by='platform', title="Sentiment Distribution by Platform (%)"
+        )
+        st.plotly_chart(platform_sentiment_pct, use_container_width=True)
+
+    # Platform metrics table
+    with st.expander("📈 Detailed Platform Metrics"):
+        platform_metrics = SentimentMetrics.calculate_platform_metrics(df)
+
+        platform_data = []
+        for platform, metrics in platform_metrics.items():
+            platform_data.append({
+                'Platform': platform.title(),
+                'Total Comments': metrics['total_comments'],
+                'Replies Needed': metrics['total_reply_required'],
+                'Negative %': f"{metrics['negative_pct']:.1f}%",
+                'Positive %': f"{metrics['positive_pct']:.1f}%",
+                'Avg Sentiment Score': f"{metrics['avg_sentiment_score']:.2f}"
+            })
+
+        st.table(platform_data)
+
+    st.markdown("---")
+
+    # Intent Analysis
+    st.markdown("## 🎭 Intent Analysis")
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        # Intent bar chart
+        intent_bar = distribution_charts.create_intent_bar_chart(
+            df, title="Intent Distribution", orientation='h'
+        )
+        st.plotly_chart(intent_bar, use_container_width=True)
+
+    with col2:
+        # Intent pie chart
+        intent_pie = distribution_charts.create_intent_pie_chart(df, title="Intent Distribution")
+        st.plotly_chart(intent_pie, use_container_width=True)
+
+    st.markdown("---")
+
+    # Brand-Platform Matrix
+    st.markdown("## 🔀 Cross-Dimensional Analysis")
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        # Heatmap showing comment distribution
+        brand_platform_matrix = distribution_charts.create_brand_platform_matrix(
+            df, title="Brand-Platform Comment Matrix"
+        )
+        st.plotly_chart(brand_platform_matrix, use_container_width=True)
+
+    with col2:
+        # Sentiment heatmap
+        sentiment_heatmap = sentiment_charts.create_sentiment_heatmap(
+            df, row_dimension='brand', col_dimension='platform', title="Negative Sentiment Heatmap"
+        )
+        st.plotly_chart(sentiment_heatmap, use_container_width=True)
+
+    st.markdown("---")
+
+    # Platform and Brand Distribution
+    st.markdown("## 📊 Volume Analysis")
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        # Platform distribution
+        platform_dist = distribution_charts.create_platform_distribution(df, title="Comments by Platform")
+        st.plotly_chart(platform_dist, use_container_width=True)
+
+    with col2:
+        # Brand distribution
+        brand_dist = distribution_charts.create_brand_distribution(df, title="Comments by Brand")
+        st.plotly_chart(brand_dist, use_container_width=True)
+
+    st.markdown("---")
+
+    # Reply Requirements
+    st.markdown("## ⚠️ Reply Requirements Analysis")
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        # Reply required by brand
+        reply_brand = distribution_charts.create_reply_required_chart(
+            df, group_by='brand', title="Comments Requiring Reply by Brand"
+        )
+        st.plotly_chart(reply_brand, use_container_width=True)
+
+    with col2:
+        # Reply required by platform
+        reply_platform = distribution_charts.create_reply_required_chart(
+            df, group_by='platform', title="Comments Requiring Reply by Platform"
+        )
+        st.plotly_chart(reply_platform, use_container_width=True)
+
+    # Response urgency metrics
+    urgency_metrics = SentimentMetrics.calculate_response_urgency(df)
+
+    st.markdown("### 🚨 Response Urgency Breakdown")
+    urgency_col1, urgency_col2, urgency_col3, urgency_col4 = st.columns(4)
+
+    with urgency_col1:
+        st.metric("🔴 Urgent", urgency_metrics['urgent_count'], help="Negative sentiment + requires reply")
+
+    with urgency_col2:
+        st.metric("🟠 High Priority", urgency_metrics['high_priority_count'], help="Neutral with feedback/request")
+
+    with urgency_col3:
+        st.metric("🟡 Medium Priority", urgency_metrics['medium_priority_count'], help="Positive requiring reply")
+
+    with urgency_col4:
+        st.metric("🟢 Low Priority", urgency_metrics['low_priority_count'], help="Very positive requiring reply")
+
+    st.markdown("---")
+
+    st.markdown("---")
+
+    # Demographics Analysis (for musora_app only)
+    # Check if we have musora_app data and demographic fields
+    has_musora_app = 'platform' in df.columns and 'musora_app' in df['platform'].values
+    has_demographics = (
+        has_musora_app and
+        'age_group' in df.columns and
+        'timezone' in df.columns and
+        'experience_level' in df.columns
+    )
+
+    if has_demographics:
+        # Filter for musora_app data only
+        df_musora = df[df['platform'] == 'musora_app'].copy()
+
+        # Check if we have any demographic data (not all Unknown)
+        has_valid_demographics = (
+            (df_musora['age_group'] != 'Unknown').any() or
+            (df_musora['timezone_region'] != 'Unknown').any() or
+            (df_musora['experience_group'] != 'Unknown').any()
+        )
+
+        if has_valid_demographics and len(df_musora) > 0:
+            st.markdown("## 👥 Demographics Analysis (Musora App)")
+            st.info(f"📊 Analyzing demographics for **{len(df_musora):,}** Musora App comments")
+
+            # Initialize demographic charts
+            demographic_charts = DemographicCharts()
+
+            # Get demographic summary
+            demo_summary = processor.get_demographics_summary(df_musora)
+
+            # Display summary metrics
+            demo_col1, demo_col2, demo_col3, demo_col4 = st.columns(4)
+
+            with demo_col1:
+                st.metric(
+                    "Comments with Demographics",
+                    f"{demo_summary['users_with_demographics']:,}",
+                    f"{demo_summary['coverage_percentage']:.1f}% coverage"
+                )
+
+            with demo_col2:
+                if demo_summary['avg_age'] is not None:
+                    st.metric("Average Age", f"{demo_summary['avg_age']:.1f} years")
+                else:
+                    st.metric("Average Age", "N/A")
+
+            with demo_col3:
+                st.metric("Most Common Region", demo_summary['most_common_region'])
+
+            with demo_col4:
+                if demo_summary['avg_experience'] is not None:
+                    st.metric("Avg Experience", f"{demo_summary['avg_experience']:.1f}/10")
+                else:
+                    st.metric("Avg Experience", "N/A")
+
+            st.markdown("---")
+
+            # Age Analysis
+            st.markdown("### 🎂 Age Distribution")
+
+            age_dist = processor.get_demographics_distribution(df_musora, 'age_group')
+            age_sentiment = processor.get_demographics_by_sentiment(df_musora, 'age_group')
+
+            if not age_dist.empty:
+                col1, col2 = st.columns(2)
+
+                with col1:
+                    age_chart = demographic_charts.create_age_distribution_chart(
+                        age_dist,
+                        title="Comments by Age Group"
+                    )
+                    st.plotly_chart(age_chart, use_container_width=True)
+
+                with col2:
+                    age_sent_chart = demographic_charts.create_age_sentiment_chart(
+                        age_sentiment,
+                        title="Sentiment Distribution by Age Group"
+                    )
+                    st.plotly_chart(age_sent_chart, use_container_width=True)
+
+                # Insights
+                with st.expander("💡 Age Insights"):
+                    if len(age_dist) > 0:
+                        top_age_group = age_dist.iloc[0]['age_group']
+                        top_age_count = age_dist.iloc[0]['count']
+                        top_age_pct = age_dist.iloc[0]['percentage']
+
+                        st.write(f"**Most Active Age Group:** {top_age_group} ({top_age_count:,} comments, {top_age_pct:.1f}%)")
+
+                        # Find age group with most negative sentiment
+                        if not age_sentiment.empty:
+                            negative_sentiments = age_sentiment[
+                                age_sentiment['sentiment_polarity'].isin(['negative', 'very_negative'])
+                            ].groupby('age_group')['percentage'].sum().reset_index()
+
+                            if len(negative_sentiments) > 0:
+                                negative_sentiments = negative_sentiments.sort_values('percentage', ascending=False)
+                                most_negative_age = negative_sentiments.iloc[0]['age_group']
+                                most_negative_pct = negative_sentiments.iloc[0]['percentage']
+                                st.write(f"**Highest Negative Sentiment:** {most_negative_age} ({most_negative_pct:.1f}% negative)")
+            else:
+                st.info("No age data available for visualization")
+
+            st.markdown("---")
+
+            # Timezone Analysis
+            st.markdown("### 🌍 Geographic Distribution")
+
+            # Get timezone data
+            top_timezones = processor.get_top_timezones(df_musora, top_n=15)
+            region_dist = processor.get_timezone_regions_distribution(df_musora)
+            region_sentiment = processor.get_demographics_by_sentiment(df_musora, 'timezone_region')
+
+            if not top_timezones.empty or not region_dist.empty:
+                # Top timezones
+                if not top_timezones.empty:
+                    st.markdown("#### Top 15 Timezones")
+                    timezone_chart = demographic_charts.create_timezone_chart(
+                        top_timezones,
+                        title="Most Common Timezones",
+                        top_n=15
+                    )
+                    st.plotly_chart(timezone_chart, use_container_width=True)
+
+                # Regional distribution
+                if not region_dist.empty:
+                    st.markdown("#### Regional Distribution")
+                    col1, col2 = st.columns(2)
+
+                    with col1:
+                        region_chart = demographic_charts.create_region_distribution_chart(
+                            region_dist,
+                            title="Comments by Region"
+                        )
+                        st.plotly_chart(region_chart, use_container_width=True)
+
+                    with col2:
+                        if not region_sentiment.empty:
+                            region_sent_chart = demographic_charts.create_region_sentiment_chart(
+                                region_sentiment,
+                                title="Sentiment Distribution by Region"
+                            )
+                            st.plotly_chart(region_sent_chart, use_container_width=True)
+
+                # Insights
+                with st.expander("💡 Geographic Insights"):
+                    if not top_timezones.empty:
+                        top_tz = top_timezones.iloc[0]['timezone']
+                        top_tz_count = top_timezones.iloc[0]['count']
+                        top_tz_pct = top_timezones.iloc[0]['percentage']
+                        st.write(f"**Most Common Timezone:** {top_tz} ({top_tz_count:,} comments, {top_tz_pct:.1f}%)")
+
+                    if not region_dist.empty:
+                        top_region = region_dist.iloc[0]['timezone_region']
+                        top_region_count = region_dist.iloc[0]['count']
+                        top_region_pct = region_dist.iloc[0]['percentage']
+                        st.write(f"**Most Active Region:** {top_region} ({top_region_count:,} comments, {top_region_pct:.1f}%)")
+
+                        # Find region with most negative sentiment
+                        if not region_sentiment.empty:
+                            negative_regions = region_sentiment[
+                                region_sentiment['sentiment_polarity'].isin(['negative', 'very_negative'])
+                            ].groupby('timezone_region')['percentage'].sum().reset_index()
+
+                            if len(negative_regions) > 0:
+                                negative_regions = negative_regions.sort_values('percentage', ascending=False)
+                                most_negative_region = negative_regions.iloc[0]['timezone_region']
+                                most_negative_region_pct = negative_regions.iloc[0]['percentage']
+                                st.write(f"**Highest Negative Sentiment:** {most_negative_region} ({most_negative_region_pct:.1f}% negative)")
+            else:
+                st.info("No timezone/region data available for visualization")
+
+            st.markdown("---")
+
+            # Experience Level Analysis
+            st.markdown("### 🎯 Experience Level Distribution")
+
+            # Get both detailed and grouped experience data
+            exp_dist_detailed = processor.get_experience_level_distribution(df_musora, use_groups=False)
+            exp_dist_grouped = processor.get_experience_level_distribution(df_musora, use_groups=True)
+            exp_sentiment_grouped = processor.get_demographics_by_sentiment(df_musora, 'experience_group')
+
+            if not exp_dist_detailed.empty or not exp_dist_grouped.empty:
+                # Tabs for detailed vs grouped view
+                tab1, tab2 = st.tabs(["📊 Detailed (0-10)", "📊 Grouped (Beginner/Intermediate/Advanced)"])
+
+                with tab1:
+                    if not exp_dist_detailed.empty:
+                        exp_chart_detailed = demographic_charts.create_experience_distribution_chart(
+                            exp_dist_detailed,
+                            title="Comments by Experience Level (0-10 Scale)",
+                            use_groups=False
+                        )
+                        st.plotly_chart(exp_chart_detailed, use_container_width=True)
+                    else:
+                        st.info("No detailed experience level data available")
+
+                with tab2:
+                    if not exp_dist_grouped.empty:
+                        col1, col2 = st.columns(2)
+
+                        with col1:
+                            exp_chart_grouped = demographic_charts.create_experience_distribution_chart(
+                                exp_dist_grouped,
+                                title="Comments by Experience Group",
+                                use_groups=True
+                            )
+                            st.plotly_chart(exp_chart_grouped, use_container_width=True)
+
+                        with col2:
+                            if not exp_sentiment_grouped.empty:
+                                exp_sent_chart = demographic_charts.create_experience_sentiment_chart(
+                                    exp_sentiment_grouped,
+                                    title="Sentiment by Experience Group",
+                                    use_groups=True
+                                )
+                                st.plotly_chart(exp_sent_chart, use_container_width=True)
+                    else:
+                        st.info("No grouped experience level data available")
+
+                # Insights
+                with st.expander("💡 Experience Insights"):
+                    if not exp_dist_grouped.empty:
+                        top_exp_group = exp_dist_grouped.iloc[0]['experience_group']
+                        top_exp_count = exp_dist_grouped.iloc[0]['count']
+                        top_exp_pct = exp_dist_grouped.iloc[0]['percentage']
+                        st.write(f"**Most Active Group:** {top_exp_group} ({top_exp_count:,} comments, {top_exp_pct:.1f}%)")
+
+                        # Find experience group with most negative sentiment
+                        if not exp_sentiment_grouped.empty:
+                            negative_exp = exp_sentiment_grouped[
+                                exp_sentiment_grouped['sentiment_polarity'].isin(['negative', 'very_negative'])
+                            ].groupby('experience_group')['percentage'].sum().reset_index()
+
+                            if len(negative_exp) > 0:
+                                negative_exp = negative_exp.sort_values('percentage', ascending=False)
+                                most_negative_exp = negative_exp.iloc[0]['experience_group']
+                                most_negative_exp_pct = negative_exp.iloc[0]['percentage']
+                                st.write(f"**Highest Negative Sentiment:** {most_negative_exp} ({most_negative_exp_pct:.1f}% negative)")
+
+                    if demo_summary['avg_experience'] is not None:
+                        st.write(f"**Average Experience Level:** {demo_summary['avg_experience']:.2f}/10")
+                        st.write(f"**Most Common Experience Group:** {demo_summary.get('most_common_experience', 'Unknown')}")
+            else:
+                st.info("No experience level data available for visualization")
+
+            st.markdown("---")
+
+    # Language Distribution (if available)
+    if 'detected_language' in df.columns:
+        st.markdown("## 🌍 Language Distribution")
+
+        lang_dist = distribution_charts.create_language_distribution(df, top_n=10, title="Top 10 Languages")
+        st.plotly_chart(lang_dist, use_container_width=True)
+
+    st.markdown("---")
+
+    # Temporal trends (if timestamp available)
+    if 'comment_timestamp' in df.columns and not df.empty:
+        with st.expander("📈 Temporal Trends", expanded=False):
+            # Frequency selector
+            freq_col1, freq_col2 = st.columns([1, 3])
+
+            with freq_col1:
+                freq = st.selectbox(
+                    "Time Granularity",
+                    options=['D', 'W', 'M'],
+                    format_func=lambda x: {'D': 'Daily', 'W': 'Weekly', 'M': 'Monthly'}[x],
+                    index=1  # Default to Weekly
+                )
+
+            sentiment_timeline = sentiment_charts.create_sentiment_timeline(df, freq=freq, title="Sentiment Trends Over Time")
+            st.plotly_chart(sentiment_timeline, use_container_width=True)
+
+    # Hierarchical sunburst
+    with st.expander("🌟 Hierarchical View", expanded=False):
+        st.markdown("**Interactive Brand > Platform > Sentiment Distribution**")
+        sunburst = distribution_charts.create_combined_distribution_sunburst(
+            df, title="Brand > Platform > Sentiment Distribution"
+        )
+        st.plotly_chart(sunburst, use_container_width=True)

visualization/components/reply_required.py

@@ -0,0 +1,324 @@
+"""
+Reply Required Page
+Displays comments that require replies with filtering and prioritization
+"""
+import streamlit as st
+import pandas as pd
+import sys
+from pathlib import Path
+
+# Add parent directory to path
+parent_dir = Path(__file__).resolve().parent.parent
+sys.path.append(str(parent_dir))
+
+from utils.data_processor import SentimentDataProcessor
+from utils.metrics import SentimentMetrics
+from visualizations.sentiment_charts import SentimentCharts
+from visualizations.distribution_charts import DistributionCharts
+from visualizations.content_cards import ContentCards
+
+
+def render_reply_required(df):
+    """
+    Render the reply required page
+
+    Args:
+        df: Sentiment dataframe
+    """
+    st.title("⚠️ Comments Requiring Reply")
+    st.markdown("Manage and prioritize comments that need responses")
+    st.markdown("---")
+
+    # Initialize components
+    processor = SentimentDataProcessor()
+    metrics = SentimentMetrics()
+
+    # Get comments requiring reply
+    reply_comments = processor.get_comments_requiring_reply(df)
+
+    if reply_comments.empty:
+        st.success("🎉 Great news! No comments currently require replies.")
+        return
+
+    # Display summary statistics
+    st.markdown("### 📊 Summary")
+
+    col1, col2, col3, col4 = st.columns(4)
+
+    with col1:
+        st.metric("Total Replies Needed", len(reply_comments))
+
+    with col2:
+        urgency = metrics.calculate_response_urgency(df)
+        st.metric("🔴 Urgent", urgency['urgent_count'], help="Negative sentiment")
+
+    with col3:
+        unique_contents = reply_comments['content_sk'].nunique() if 'content_sk' in reply_comments.columns else 0
+        st.metric("Affected Contents", unique_contents)
+
+    with col4:
+        negative_count = reply_comments['sentiment_polarity'].isin(['negative', 'very_negative']).sum()
+        negative_pct = (negative_count / len(reply_comments) * 100) if len(reply_comments) > 0 else 0
+        st.metric("Negative %", f"{negative_pct:.1f}%")
+
+    st.markdown("---")
+
+    # Urgency breakdown
+    st.markdown("### 🚨 Response Urgency Breakdown")
+
+    urgency_metrics = metrics.calculate_response_urgency(df)
+
+    urgency_col1, urgency_col2, urgency_col3, urgency_col4 = st.columns(4)
+
+    with urgency_col1:
+        st.metric(
+            "🔴 Urgent",
+            urgency_metrics['urgent_count'],
+            help="Negative sentiment requiring reply - immediate action needed"
+        )
+
+    with urgency_col2:
+        st.metric(
+            "🟠 High Priority",
+            urgency_metrics['high_priority_count'],
+            help="Neutral with feedback/request - respond within 24 hours"
+        )
+
+    with urgency_col3:
+        st.metric(
+            "🟡 Medium Priority",
+            urgency_metrics['medium_priority_count'],
+            help="Positive requiring reply - respond within 48 hours"
+        )
+
+    with urgency_col4:
+        st.metric(
+            "🟢 Low Priority",
+            urgency_metrics['low_priority_count'],
+            help="Very positive requiring reply - respond when convenient"
+        )
+
+    st.markdown("---")
+
+    # Filters
+    st.markdown("### 🔍 Filters")
+
+    filter_col1, filter_col2, filter_col3, filter_col4 = st.columns(4)
+
+    with filter_col1:
+        # Priority filter
+        priority_options = ['All', '🔴 Urgent', '🟠 High', '🟡 Medium', '🟢 Low']
+        selected_priority = st.selectbox("Priority", priority_options, index=0)
+
+    with filter_col2:
+        # Platform filter
+        platform_options = ['All'] + sorted(reply_comments['platform'].unique().tolist())
+        selected_platform = st.selectbox("Platform", platform_options, index=0)
+
+    with filter_col3:
+        # Brand filter
+        brand_options = ['All'] + sorted(reply_comments['brand'].unique().tolist())
+        selected_brand = st.selectbox("Brand", brand_options, index=0)
+
+    with filter_col4:
+        # Intent filter
+        intent_list = reply_comments['intent'].str.split(',').explode().str.strip().unique().tolist()
+        intent_options = ['All'] + sorted(intent_list)
+        selected_intent = st.selectbox("Intent", intent_options, index=0)
+
+    # Apply filters
+    filtered_comments = reply_comments.copy()
+
+    # Priority filtering
+    if selected_priority != 'All':
+        if selected_priority == '🔴 Urgent':
+            filtered_comments = filtered_comments[
+                filtered_comments['sentiment_polarity'].isin(['negative', 'very_negative'])
+            ]
+        elif selected_priority == '🟠 High':
+            filtered_comments = filtered_comments[
+                (filtered_comments['sentiment_polarity'] == 'neutral') &
+                (filtered_comments['intent'].str.contains('feedback_negative|request', na=False))
+            ]
+        elif selected_priority == '🟡 Medium':
+            filtered_comments = filtered_comments[
+                filtered_comments['sentiment_polarity'] == 'positive'
+            ]
+        elif selected_priority == '🟢 Low':
+            filtered_comments = filtered_comments[
+                filtered_comments['sentiment_polarity'] == 'very_positive'
+            ]
+
+    # Platform filtering
+    if selected_platform != 'All':
+        filtered_comments = filtered_comments[filtered_comments['platform'] == selected_platform]
+
+    # Brand filtering
+    if selected_brand != 'All':
+        filtered_comments = filtered_comments[filtered_comments['brand'] == selected_brand]
+
+    # Intent filtering
+    if selected_intent != 'All':
+        filtered_comments = filtered_comments[
+            filtered_comments['intent'].str.contains(selected_intent, na=False)
+        ]
+
+    st.markdown(f"**Showing {len(filtered_comments)} comments after filtering**")
+
+    st.markdown("---")
+
+    # Visualizations
+    if not filtered_comments.empty:
+        st.markdown("### 📈 Analysis")
+
+        viz_col1, viz_col2 = st.columns(2)
+
+        with viz_col1:
+            # Sentiment distribution
+            sentiment_charts = SentimentCharts()
+            sentiment_pie = sentiment_charts.create_sentiment_pie_chart(
+                filtered_comments, title="Sentiment Distribution"
+            )
+            st.plotly_chart(sentiment_pie, use_container_width=True)
+
+        with viz_col2:
+            # Intent distribution
+            distribution_charts = DistributionCharts()
+            intent_bar = distribution_charts.create_intent_bar_chart(
+                filtered_comments, title="Intent Distribution", orientation='h'
+            )
+            st.plotly_chart(intent_bar, use_container_width=True)
+
+        st.markdown("---")
+
+    # Display comments
+    st.markdown("### 💬 Comments Requiring Reply")
+
+    # Pagination
+    items_per_page = 10
+    total_pages = (len(filtered_comments) - 1) // items_per_page + 1
+
+    if 'reply_page' not in st.session_state:
+        st.session_state.reply_page = 1
+
+    # Pagination controls at top
+    if total_pages > 1:
+        page_col1, page_col2, page_col3 = st.columns([1, 2, 1])
+
+        with page_col1:
+            if st.button("⬅️ Previous", key="prev_top", disabled=(st.session_state.reply_page <= 1)):
+                st.session_state.reply_page -= 1
+                st.rerun()
+
+        with page_col2:
+            st.markdown(f"<center>Page {st.session_state.reply_page} of {total_pages}</center>", unsafe_allow_html=True)
+
+        with page_col3:
+            if st.button("Next ➡️", key="next_top", disabled=(st.session_state.reply_page >= total_pages)):
+                st.session_state.reply_page += 1
+                st.rerun()
+
+        st.markdown("---")
+
+    # Get paginated comments
+    start_idx = (st.session_state.reply_page - 1) * items_per_page
+    end_idx = start_idx + items_per_page
+    paginated_comments = filtered_comments.iloc[start_idx:end_idx]
+
+    # Display comments
+    if paginated_comments.empty:
+        st.info("No comments match the selected filters")
+    else:
+        for idx, (_, comment) in enumerate(paginated_comments.iterrows(), start=start_idx + 1):
+            # Priority badge
+            priority_emoji = "🟢"
+            if comment['sentiment_polarity'] in ['negative', 'very_negative']:
+                priority_emoji = "🔴"
+            elif comment['sentiment_polarity'] == 'neutral' and any(
+                intent in comment['intent'] for intent in ['feedback_negative', 'request']
+            ):
+                priority_emoji = "🟠"
+            elif comment['sentiment_polarity'] == 'positive':
+                priority_emoji = "🟡"
+
+            st.markdown(f"#### {priority_emoji} Comment #{idx}")
+
+            # Display comment card
+            ContentCards.display_comment_card(comment, show_original=True)
+
+    # Pagination controls at bottom
+    if total_pages > 1:
+        st.markdown("---")
+
+        page_col1, page_col2, page_col3 = st.columns([1, 2, 1])
+
+        with page_col1:
+            if st.button("⬅️ Previous", key="prev_bottom", disabled=(st.session_state.reply_page <= 1)):
+                st.session_state.reply_page -= 1
+                st.rerun()
+
+        with page_col2:
+            st.markdown(f"<center>Page {st.session_state.reply_page} of {total_pages}</center>", unsafe_allow_html=True)
+
+        with page_col3:
+            if st.button("Next ➡️", key="next_bottom", disabled=(st.session_state.reply_page >= total_pages)):
+                st.session_state.reply_page += 1
+                st.rerun()
+
+    st.markdown("---")
+
+    # Export option
+    st.markdown("### 💾 Export Data")
+
+    col1, col2 = st.columns([1, 3])
+
+    with col1:
+        # Prepare export data
+        export_columns = [
+            'comment_id', 'author_name', 'platform', 'brand', 'comment_timestamp',
+            'display_text', 'original_text', 'detected_language', 'sentiment_polarity',
+            'intent', 'sentiment_confidence', 'content_description', 'permalink_url'
+        ]
+
+        # Filter only available columns
+        available_columns = [col for col in export_columns if col in filtered_comments.columns]
+        export_data = filtered_comments[available_columns]
+
+        csv = export_data.to_csv(index=False)
+
+        st.download_button(
+            label="📥 Download as CSV",
+            data=csv,
+            file_name=f"comments_requiring_reply.csv",
+            mime="text/csv"
+        )
+
+    with col2:
+        st.info("Download the filtered comments for team collaboration or CRM import")
+
+    st.markdown("---")
+
+    # Quick stats by content
+    st.markdown("### 📋 Reply Requirements by Content")
+
+    content_reply_summary = filtered_comments.groupby('content_sk').agg({
+        'comment_sk': 'count',
+        'content_description': 'first',
+        'permalink_url': 'first'
+    }).reset_index()
+
+    content_reply_summary.columns = ['content_sk', 'replies_needed', 'content_description', 'permalink_url']
+    content_reply_summary = content_reply_summary.sort_values('replies_needed', ascending=False).head(10)
+
+    for idx, (_, content) in enumerate(content_reply_summary.iterrows(), 1):
+        with st.expander(f"📝 Content #{idx} - {content['replies_needed']} replies needed"):
+            st.markdown(f"**Description:** {content['content_description']}")
+            if pd.notna(content['permalink_url']):
+                st.markdown(f"**Link:** [View Content]({content['permalink_url']})")
+
+            # Show comments for this content
+            content_comments = filtered_comments[filtered_comments['content_sk'] == content['content_sk']].head(3)
+
+            st.markdown(f"**Top {len(content_comments)} comments:**")
+            for _, comment in content_comments.iterrows():
+                ContentCards.display_comment_card(comment, show_original=True)

visualization/components/sentiment_analysis.py

@@ -0,0 +1,671 @@
+"""
+Sentiment Analysis Page
+Analyze content performance across all sentiment types with advanced filtering
+"""
+import streamlit as st
+import sys
+from pathlib import Path
+
+# Add parent directory to path
+parent_dir = Path(__file__).resolve().parent.parent
+sys.path.append(str(parent_dir))
+
+from utils.data_processor import SentimentDataProcessor
+from visualizations.sentiment_charts import SentimentCharts
+from visualizations.distribution_charts import DistributionCharts
+from visualizations.content_cards import ContentCards
+from agents.content_summary_agent import ContentSummaryAgent
+
+
+def render_sentiment_analysis(df):
+    """
+    Render the sentiment analysis page
+
+    Args:
+        df: Sentiment dataframe (full dataset from app.py)
+    """
+    st.title("🔍 Sentiment Analysis")
+    st.markdown("Analyze content performance based on sentiment patterns and user feedback")
+    st.markdown("---")
+
+    # Initialize components
+    processor = SentimentDataProcessor()
+    sentiment_charts = SentimentCharts()
+    distribution_charts = DistributionCharts()
+
+    # Initialize AI agent
+    summary_agent = ContentSummaryAgent(model="gpt-5-nano", temperature=1)
+
+    # Initialize session state for caching summaries
+    if 'content_summaries' not in st.session_state:
+        st.session_state.content_summaries = {}
+
+    # Page-Specific Filters (Platform and Brand Selection)
+    st.markdown("### 🎯 Select Platform and Brand")
+    st.info("⚡ **Performance Optimization**: Select a specific platform and brand to analyze. This filters the data and makes the page load faster.")
+
+    filter_col1, filter_col2 = st.columns(2)
+
+    with filter_col1:
+        # Get available platforms
+        available_platforms = sorted(df['platform'].unique().tolist())
+        selected_platform = st.selectbox(
+            "Platform *",
+            options=[''] + available_platforms,
+            index=0,
+            help="Select the platform to analyze"
+        )
+
+    with filter_col2:
+        # Get available brands
+        available_brands = sorted(df['brand'].unique().tolist())
+        selected_brand = st.selectbox(
+            "Brand *",
+            options=[''] + available_brands,
+            index=0,
+            help="Select the brand to analyze"
+        )
+
+    # Check if both platform and brand are selected
+    if not selected_platform or not selected_brand:
+        st.warning("⚠️ Please select both **Platform** and **Brand** to view sentiment analysis.")
+        st.markdown("---")
+
+        # Show summary of available data
+        st.markdown("### 📊 Available Data Summary")
+        col1, col2, col3 = st.columns(3)
+
+        with col1:
+            st.metric("Total Comments", f"{len(df):,}")
+
+        with col2:
+            st.metric("Platforms", len(available_platforms))
+            with st.expander("View Platforms"):
+                for platform in available_platforms:
+                    count = len(df[df['platform'] == platform])
+                    st.write(f"- **{platform}**: {count:,} comments")
+
+        with col3:
+            st.metric("Brands", len(available_brands))
+            with st.expander("View Brands"):
+                for brand in available_brands:
+                    count = len(df[df['brand'] == brand])
+                    st.write(f"- **{brand}**: {count:,} comments")
+
+        return
+
+    # Filter dataframe by selected platform and brand
+    df_filtered = df[
+        (df['platform'] == selected_platform) &
+        (df['brand'] == selected_brand)
+    ].copy()
+
+    if df_filtered.empty:
+        st.error(f"❌ No data found for **{selected_platform}** + **{selected_brand}** combination.")
+        return
+
+    # Show data info
+    st.success(f"✅ Loaded **{len(df_filtered):,}** comments for **{selected_platform}** + **{selected_brand}**")
+    st.markdown("---")
+
+    # Filters Section
+    st.markdown("### 🔍 Content Filters")
+    st.markdown("Filter contents by sentiment and intent to focus your analysis")
+
+    filter_col1, filter_col2, filter_col3, filter_col4 = st.columns(4)
+
+    with filter_col1:
+        # Sentiment filter
+        sentiment_options = df_filtered['sentiment_polarity'].unique().tolist()
+        selected_sentiments = st.multiselect(
+            "Sentiment",
+            options=sorted(sentiment_options),
+            default=[],
+            help="Filter by dominant sentiment. Leave empty to show all sentiments."
+        )
+
+    with filter_col2:
+        # Intent filter
+        intent_list = df_filtered['intent'].str.split(',').explode().str.strip().unique().tolist()
+        selected_intents = st.multiselect(
+            "Intent",
+            options=sorted([i for i in intent_list if i]),
+            default=[],
+            help="Filter contents that have comments with these intents"
+        )
+
+    with filter_col3:
+        # Top N selector
+        top_n = st.selectbox(
+            "Top N Contents",
+            options=[5, 10, 15, 20, 25],
+            index=1,  # Default to 10
+            help="Number of contents to display"
+        )
+
+    with filter_col4:
+        # Show filter status
+        filter_active = bool(selected_sentiments or selected_intents)
+        if filter_active:
+            st.metric("Filters Active", "✓ Yes", help="Sentiment or intent filters are applied")
+        else:
+            st.metric("Filters Active", "✗ No", help="Showing all sentiments")
+
+    st.markdown("---")
+
+    # Advanced controls (defaults optimized for balanced analysis)
+    with st.expander("⚙️ Advanced Ranking Controls", expanded=False):
+        st.markdown("**Customize how contents are ranked and filtered**")
+
+        adv_col1, adv_col2 = st.columns(2)
+
+        with adv_col1:
+            min_comments = st.slider(
+                "Minimum Comments Required",
+                min_value=1,
+                max_value=50,
+                value=10,
+                step=1,
+                help="Filter out contents with fewer comments. Default: 10 (excludes low-volume contents)"
+            )
+
+        with adv_col2:
+            sort_by = st.selectbox(
+                "Sort By",
+                options=[
+                    ('severity_score', '🎯 Severity Score (Balanced) - Recommended'),
+                    ('sentiment_percentage', '📊 Sentiment Percentage'),
+                    ('sentiment_count', '🔢 Sentiment Count (Absolute)'),
+                    ('total_comments', '💬 Total Comments (Volume)')
+                ],
+                format_func=lambda x: x[1],
+                index=0,
+                help="Severity Score balances sentiment % with comment volume for smarter ranking"
+            )
+            sort_by_value = sort_by[0]
+
+        # Show explanation of current sort method
+        sentiment_label = "selected sentiments" if selected_sentiments else "all sentiments"
+        if sort_by_value == 'severity_score':
+            st.info(f"📘 **Severity Score** = Sentiment % × √(Total Comments). Balances {sentiment_label} percentage with volume for high-impact ranking.")
+        elif sort_by_value == 'sentiment_percentage':
+            st.info(f"📘 Ranks by highest % of {sentiment_label}. May include low-volume contents.")
+        elif sort_by_value == 'sentiment_count':
+            st.info(f"📘 Ranks by absolute number of comments with {sentiment_label}. Prioritizes volume over percentage.")
+        else:
+            st.info("📘 Ranks by total comment volume, regardless of sentiment.")
+
+    # Get filtered contents using the new method
+    filtered_contents = processor.get_sentiment_filtered_contents(
+        df_filtered,
+        selected_sentiments=selected_sentiments if selected_sentiments else None,
+        selected_intents=selected_intents if selected_intents else None,
+        top_n=top_n,
+        min_comments=min_comments,
+        sort_by=sort_by_value
+    )
+
+    # Reset pagination when filters change
+    filter_key = f"{selected_platform}_{selected_brand}_{top_n}_{min_comments}_{sort_by_value}_{str(selected_sentiments)}_{str(selected_intents)}"
+    if 'last_filter_key' not in st.session_state or st.session_state.last_filter_key != filter_key:
+        st.session_state.sentiment_page = 1
+        st.session_state.last_filter_key = filter_key
+
+    if filtered_contents.empty:
+        st.warning("No content data available with the selected filters. Try adjusting your filters.")
+        return
+
+    # Display summary statistics
+    st.markdown("### 📊 Summary")
+
+    col1, col2, col3, col4 = st.columns(4)
+
+    with col1:
+        st.metric("Contents Analyzed", len(filtered_contents))
+
+    with col2:
+        # Dynamic metric based on selected sentiments
+        if 'selected_sentiment_percentage' in filtered_contents.columns:
+            avg_sentiment_pct = filtered_contents['selected_sentiment_percentage'].mean()
+            sentiment_label = "Selected Sentiment %" if selected_sentiments else "All Sentiment %"
+            st.metric(sentiment_label, f"{avg_sentiment_pct:.1f}%")
+        else:
+            avg_negative_pct = filtered_contents['negative_percentage'].mean()
+            st.metric("Avg Negative %", f"{avg_negative_pct:.1f}%")
+
+    with col3:
+        total_comments = filtered_contents['total_comments'].sum()
+        st.metric("Total Comments", int(total_comments))
+
+    with col4:
+        total_replies_needed = filtered_contents['reply_required_count'].sum()
+        st.metric("Total Replies Needed", int(total_replies_needed))
+
+    st.markdown("---")
+
+    # Engagement scatter plot
+    st.markdown("### 📈 Content Engagement Analysis")
+    engagement_scatter = distribution_charts.create_engagement_scatter(
+        filtered_contents, title="Content Engagement vs. Sentiment"
+    )
+    st.plotly_chart(engagement_scatter, use_container_width=True, key="engagement_scatter_chart")
+
+    st.markdown("---")
+
+    # Display each content with detailed analysis
+    st.markdown("### 🔍 Detailed Content Analysis")
+
+    # Add pagination controls for better performance
+    if 'sentiment_page' not in st.session_state:
+        st.session_state.sentiment_page = 1
+
+    items_per_page = 5  # Show 5 contents per page
+    total_contents = len(filtered_contents)
+    total_pages = (total_contents + items_per_page - 1) // items_per_page  # Ceiling division
+
+    if total_contents > items_per_page:
+        # Display pagination info
+        st.info(f"📄 Showing page {st.session_state.sentiment_page} of {total_pages} ({total_contents} total contents)")
+
+        # Pagination controls at top
+        col_prev, col_info, col_next = st.columns([1, 2, 1])
+
+        with col_prev:
+            if st.button("⬅️ Previous", key="prev_top", disabled=st.session_state.sentiment_page == 1):
+                st.session_state.sentiment_page -= 1
+                st.rerun()
+
+        with col_info:
+            st.markdown(f"<div style='text-align: center; padding-top: 8px;'>Page {st.session_state.sentiment_page} / {total_pages}</div>", unsafe_allow_html=True)
+
+        with col_next:
+            if st.button("Next ➡️", key="next_top", disabled=st.session_state.sentiment_page >= total_pages):
+                st.session_state.sentiment_page += 1
+                st.rerun()
+
+        st.markdown("---")
+
+    # Calculate pagination indices
+    start_idx = (st.session_state.sentiment_page - 1) * items_per_page
+    end_idx = min(start_idx + items_per_page, total_contents)
+
+    # Get paginated contents
+    paginated_contents = filtered_contents.iloc[start_idx:end_idx]
+
+    for idx, (_, content_row) in enumerate(paginated_contents.iterrows(), start_idx + 1):
+        # Display content card
+        ContentCards.display_content_card(content_row, rank=idx)
+
+        # Get comments for this content
+        content_comments = df_filtered[df_filtered['content_sk'] == content_row['content_sk']]
+
+        if content_comments.empty:
+            st.info("No comment details available for this content")
+            continue
+
+        # Create columns for visualizations
+        viz_col1, viz_col2 = st.columns(2)
+
+        with viz_col1:
+            # Sentiment distribution for this content
+            content_sentiment_pie = sentiment_charts.create_sentiment_pie_chart(
+                content_comments, title=f"Sentiment Distribution"
+            )
+            st.plotly_chart(content_sentiment_pie, use_container_width=True, key=f"sentiment_pie_{content_row['content_sk']}")
+
+        with viz_col2:
+            # Intent distribution for this content
+            content_intent_bar = distribution_charts.create_intent_bar_chart(
+                content_comments, title=f"Intent Distribution", orientation='h'
+            )
+            st.plotly_chart(content_intent_bar, use_container_width=True, key=f"intent_bar_{content_row['content_sk']}")
+
+        # AI Analysis Section
+        st.markdown("#### 🤖 AI-Powered Analysis")
+
+        content_sk = content_row['content_sk']
+
+        # Three buttons in a row for different analysis types
+        st.markdown("**Select analysis type:**")
+        btn_col1, btn_col2, btn_col3 = st.columns(3)
+
+        with btn_col1:
+            generate_negative = st.button(
+                "📉 Negative Summary",
+                key=f"ai_negative_{content_sk}",
+                help="Analyze negative comments only",
+                use_container_width=True
+            )
+
+        with btn_col2:
+            generate_combined = st.button(
+                "📊 Combined Summary",
+                key=f"ai_combined_{content_sk}",
+                help="Analyze both positive and negative comments",
+                use_container_width=True
+            )
+
+        with btn_col3:
+            generate_positive = st.button(
+                "📈 Positive Summary",
+                key=f"ai_positive_{content_sk}",
+                help="Analyze positive comments only",
+                use_container_width=True
+            )
+
+        # Determine which summary to generate/display
+        summary_type = None
+        if generate_negative:
+            summary_type = 'negative'
+        elif generate_positive:
+            summary_type = 'positive'
+        elif generate_combined:
+            summary_type = 'combined'
+
+        # Check if any summary already exists in session state
+        summary_key_negative = f"{content_sk}_negative"
+        summary_key_positive = f"{content_sk}_positive"
+        summary_key_combined = f"{content_sk}_combined"
+
+        # Display existing summaries or generate new ones
+        if summary_type or summary_key_negative in st.session_state.content_summaries or \
+           summary_key_positive in st.session_state.content_summaries or \
+           summary_key_combined in st.session_state.content_summaries:
+
+            # Generate new summary if button was clicked
+            if summary_type:
+                summary_key = f"{content_sk}_{summary_type}"
+                with st.spinner(f"Analyzing {summary_type} comments with AI..."):
+                    # Prepare input for agent
+                    agent_input = {
+                        'content_sk': content_sk,
+                        'content_description': content_row['content_description'],
+                        'comments': content_comments,
+                        'sentiment_type': summary_type
+                    }
+
+                    # Generate summary
+                    result = summary_agent.process(agent_input)
+
+                    # Cache the result
+                    st.session_state.content_summaries[summary_key] = result
+
+            # Display all available summaries
+            available_summaries = []
+            if summary_key_negative in st.session_state.content_summaries:
+                available_summaries.append(('Negative', summary_key_negative))
+            if summary_key_combined in st.session_state.content_summaries:
+                available_summaries.append(('Combined', summary_key_combined))
+            if summary_key_positive in st.session_state.content_summaries:
+                available_summaries.append(('Positive', summary_key_positive))
+
+            # Display each available summary
+            for summary_label, summary_key in available_summaries:
+                result = st.session_state.content_summaries[summary_key]
+
+                # Display the summary
+                if result['success']:
+                    summary = result['summary']
+
+                    with st.expander(f"📊 AI Analysis Report - {summary_label}", expanded=True):
+                        # Executive Summary
+                        st.markdown("### Executive Summary")
+                        st.info(summary['executive_summary'])
+
+                        # Main Themes
+                        if summary['main_themes']:
+                            st.markdown("### 🎯 Main Themes")
+                            for theme in summary['main_themes']:
+                                sentiment_emoji = {
+                                    'positive': '😊',
+                                    'negative': '😟',
+                                    'mixed': '🤔'
+                                }.get(theme.get('sentiment', 'mixed'), '🤔')
+
+                                st.markdown(f"""
+**{sentiment_emoji} {theme.get('theme', 'Unknown')}** ({theme.get('sentiment', 'mixed').title()})
+- {theme.get('description', 'No description')}
+""")
+
+                        # Two-column layout for praise and complaints
+                        col_praise, col_complaints = st.columns(2)
+
+                        with col_praise:
+                            st.markdown("### ✅ Praise Points")
+                            if summary['praise_points']:
+                                for point in summary['praise_points']:
+                                    st.markdown(f"- {point}")
+                            else:
+                                st.markdown("*No significant praise points identified*")
+
+                        with col_complaints:
+                            st.markdown("### ⚠️ Key Complaints")
+                            if summary['key_complaints']:
+                                for complaint in summary['key_complaints']:
+                                    st.markdown(f"- {complaint}")
+                            else:
+                                st.markdown("*No significant complaints identified*")
+
+                        # FAQs and Insights
+                        col_faq, col_insights = st.columns(2)
+
+                        with col_faq:
+                            st.markdown("### ❓ Frequently Asked Questions")
+                            if summary['frequently_asked_questions']:
+                                for faq in summary['frequently_asked_questions']:
+                                    st.markdown(f"- {faq}")
+                            else:
+                                st.markdown("*No frequent questions identified*")
+
+                        with col_insights:
+                            st.markdown("### 💡 Unexpected Insights")
+                            if summary['unexpected_insights']:
+                                for insight in summary['unexpected_insights']:
+                                    st.markdown(f"- {insight}")
+                            else:
+                                st.markdown("*No unexpected insights identified*")
+
+                        # Action Recommendations
+                        if summary['action_recommendations']:
+                            st.markdown("### 🎯 Recommended Actions")
+                            for action in summary['action_recommendations']:
+                                priority = action.get('priority', 'medium').upper()
+                                priority_color = {
+                                    'HIGH': '🔴',
+                                    'MEDIUM': '🟡',
+                                    'LOW': '🟢'
+                                }.get(priority, '🟡')
+
+                                st.markdown(f"{priority_color} **[{priority}]** {action.get('action', 'No action specified')}")
+
+                        # Metadata
+                        with st.expander("ℹ️ Analysis Metadata"):
+                            metadata = result.get('metadata', {})
+                            meta_col1, meta_col2, meta_col3 = st.columns(3)
+
+                            with meta_col1:
+                                st.metric("Comments Analyzed", metadata.get('total_comments_analyzed', 0))
+
+                            with meta_col2:
+                                st.metric("Model Used", metadata.get('model_used', 'N/A'))
+
+                            with meta_col3:
+                                st.metric("Tokens Used", metadata.get('tokens_used', 0))
+
+                else:
+                    # Display error
+                    st.error(f"❌ Failed to generate AI analysis: {result.get('error', 'Unknown error')}")
+
+                    # Option to retry
+                    if st.button("🔄 Retry Analysis", key=f"retry_{summary_key}"):
+                        # Clear from cache and rerun
+                        if summary_key in st.session_state.content_summaries:
+                            del st.session_state.content_summaries[summary_key]
+                        st.rerun()
+
+        # Show comments using expandable sections - one for negative, one for positive
+        st.markdown("#### 💬 View Comments by Sentiment")
+
+        # Get negative and positive comments
+        negative_comments = content_comments[
+            content_comments['sentiment_polarity'].isin(['negative', 'very_negative'])
+        ]
+        positive_comments = content_comments[
+            content_comments['sentiment_polarity'].isin(['positive', 'very_positive'])
+        ]
+
+        negative_count = len(negative_comments)
+        positive_count = len(positive_comments)
+
+        # Create two columns for side-by-side expandable sections
+        col_neg, col_pos = st.columns(2)
+
+        with col_neg:
+            # Negative comments expandable section (collapsed by default)
+            with st.expander(f"📉 Negative Comments ({negative_count})", expanded=False):
+                if not negative_comments.empty:
+                    st.markdown(f"**Showing all {negative_count} negative comments:**")
+                    for _, comment in negative_comments.iterrows():
+                        ContentCards.display_comment_card(comment, show_original=True)
+                else:
+                    st.info("No negative comments found for this content")
+
+        with col_pos:
+            # Positive comments expandable section (collapsed by default)
+            with st.expander(f"📈 Positive Comments ({positive_count})", expanded=False):
+                if not positive_comments.empty:
+                    st.markdown(f"**Showing all {positive_count} positive comments:**")
+                    for _, comment in positive_comments.iterrows():
+                        ContentCards.display_comment_card(comment, show_original=True)
+                else:
+                    st.info("No positive comments found for this content")
+
+        st.markdown("---")
+
+    # Pagination controls at bottom
+    if total_contents > items_per_page:
+        st.markdown("---")
+
+        col_prev_bottom, col_info_bottom, col_next_bottom = st.columns([1, 2, 1])
+
+        with col_prev_bottom:
+            if st.button("⬅️ Previous", key="prev_bottom", disabled=st.session_state.sentiment_page == 1):
+                st.session_state.sentiment_page -= 1
+                st.rerun()
+
+        with col_info_bottom:
+            st.markdown(f"<div style='text-align: center; padding-top: 8px;'>Page {st.session_state.sentiment_page} / {total_pages}</div>", unsafe_allow_html=True)
+
+        with col_next_bottom:
+            if st.button("Next ➡️", key="next_bottom", disabled=st.session_state.sentiment_page >= total_pages):
+                st.session_state.sentiment_page += 1
+                st.rerun()
+
+    st.markdown("---")
+
+    # Additional insights
+    st.markdown("### 💡 Insights & Recommendations")
+
+    # Find common patterns
+    all_filtered_comments = df_filtered[df_filtered['content_sk'].isin(filtered_contents['content_sk'])]
+
+    insight_col1, insight_col2 = st.columns(2)
+
+    with insight_col1:
+        st.markdown("#### 🎯 Common Intent Patterns")
+
+        # Get intent distribution for filtered contents
+        intent_distribution = processor.get_intent_distribution(all_filtered_comments)
+        intent_distribution = intent_distribution.sort_values('count', ascending=False).head(5)
+
+        for _, intent_row in intent_distribution.iterrows():
+            st.markdown(f"- **{intent_row['intent']}**: {intent_row['count']} occurrences ({intent_row['percentage']:.1f}%)")
+
+    with insight_col2:
+        st.markdown("#### 🌐 Platform Breakdown")
+
+        # Get platform distribution
+        platform_dist = all_filtered_comments['platform'].value_counts()
+
+        for platform, count in platform_dist.items():
+            pct = (count / len(all_filtered_comments) * 100)
+            st.markdown(f"- **{platform.title()}**: {count} comments ({pct:.1f}%)")
+
+    st.markdown("---")
+
+    # Action items
+    st.markdown("### ✅ Recommended Actions")
+
+    action_items = []
+
+    # Check for high reply requirement
+    if filtered_contents['reply_required_count'].sum() > 0:
+        action_items.append(
+            f"🔴 **High Priority**: {int(filtered_contents['reply_required_count'].sum())} comments require immediate response"
+        )
+
+    # Check for critical negative percentage
+    critical_contents = filtered_contents[filtered_contents['negative_percentage'] > 50]
+    if not critical_contents.empty:
+        action_items.append(
+            f"🚨 **Critical**: {len(critical_contents)} content(s) have over 50% negative sentiment - investigate root causes"
+        )
+
+    # Check for feedback patterns
+    feedback_comments = all_filtered_comments[
+        all_filtered_comments['intent'].str.contains('feedback_negative', na=False)
+    ]
+    if not feedback_comments.empty:
+        action_items.append(
+            f"💬 **Feedback**: {len(feedback_comments)} comments contain negative feedback - consider product improvements"
+        )
+
+    # Check for questions
+    question_comments = all_filtered_comments[
+        all_filtered_comments['intent'].str.contains('question', na=False)
+    ]
+    if not question_comments.empty:
+        action_items.append(
+            f"❓ **Questions**: {len(question_comments)} unanswered questions - improve FAQ or support documentation"
+        )
+
+    if action_items:
+        for action in action_items:
+            st.markdown(action)
+    else:
+        st.success("No critical action items at this time")
+
+    st.markdown("---")
+
+    # Export option
+    st.markdown("### 💾 Export Data")
+
+    col1, col2 = st.columns([1, 3])
+
+    with col1:
+        # Prepare export data columns dynamically
+        base_columns = ['content_sk', 'content_description', 'permalink_url',
+                       'total_comments', 'reply_required_count', 'dominant_sentiment']
+
+        # Add sentiment-specific columns if they exist
+        if 'selected_sentiment_count' in filtered_contents.columns:
+            base_columns.extend(['selected_sentiment_count', 'selected_sentiment_percentage'])
+
+        if 'negative_count' in filtered_contents.columns:
+            base_columns.extend(['negative_count', 'negative_percentage'])
+
+        # Filter to only include existing columns
+        export_columns = [col for col in base_columns if col in filtered_contents.columns]
+        export_data = filtered_contents[export_columns]
+
+        csv = export_data.to_csv(index=False)
+
+        st.download_button(
+            label="📥 Download as CSV",
+            data=csv,
+            file_name=f"sentiment_analysis_top{top_n}.csv",
+            mime="text/csv"
+        )
+
+    with col2:
+        st.info("Download the data for further analysis or reporting")

visualization/config/viz_config.json

@@ -0,0 +1,87 @@
+{
+    "color_schemes": {
+        "sentiment_polarity": {
+            "very_positive": "#00C851",
+            "positive": "#7CB342",
+            "neutral": "#FFB300",
+            "negative": "#FF6F00",
+            "very_negative": "#D32F2F"
+        },
+        "intent": {
+            "praise": "#4CAF50",
+            "question": "#2196F3",
+            "request": "#9C27B0",
+            "feedback_negative": "#FF5722",
+            "suggestion": "#00BCD4",
+            "humor_sarcasm": "#FFC107",
+            "off_topic": "#9E9E9E",
+            "spam_selfpromo": "#795548"
+        },
+        "platform": {
+            "facebook": "#1877F2",
+            "instagram": "#E4405F",
+            "youtube": "#FF0000",
+            "twitter": "#1DA1F2",
+            "musora_app": "#1982C4",
+            "default": "#607D8B"
+        },
+        "brand": {
+            "drumeo": "#FF6B35",
+            "pianote": "#6A4C93",
+            "musora": "#1982C4",
+            "default": "#8AC926"
+        }
+    },
+    "sentiment_order": [
+        "very_positive",
+        "positive",
+        "neutral",
+        "negative",
+        "very_negative"
+    ],
+    "intent_order": [
+        "praise",
+        "question",
+        "request",
+        "feedback_negative",
+        "suggestion",
+        "humor_sarcasm",
+        "off_topic",
+        "spam_selfpromo"
+    ],
+    "negative_sentiments": [
+        "negative",
+        "very_negative"
+    ],
+    "dashboard": {
+        "default_date_range_days": 30,
+        "max_comments_display": 100,
+        "chart_height": 400,
+        "top_n_contents": 10
+    },
+    "page_config": {
+        "page_title": "Musora Sentiment Analysis Dashboard",
+        "page_icon": "📊",
+        "layout": "wide",
+        "initial_sidebar_state": "expanded"
+    },
+    "snowflake": {
+        "query": "SELECT s.COMMENT_SK, s.COMMENT_ID, s.ORIGINAL_TEXT, s.PLATFORM, s.COMMENT_TIMESTAMP, s.AUTHOR_NAME, s.AUTHOR_ID, CAST(NULL AS VARCHAR(16777216)) as PARENT_COMMENT_ID, CAST(NULL AS VARCHAR(16777216)) as PARENT_COMMENT_TEXT, s.CONTENT_SK, s.CONTENT_ID, s.CONTENT_DESCRIPTION, s.CHANNEL_SK, s.CHANNEL_NAME, s.CHANNEL_DISPLAY_NAME, s.DETECTED_LANGUAGE, s.LANGUAGE_CODE, s.IS_ENGLISH, s.LANGUAGE_CONFIDENCE, s.DETECTION_METHOD, s.HAS_TEXT, s.TRANSLATED_TEXT, s.TRANSLATION_PERFORMED, s.TRANSLATION_CONFIDENCE, s.TRANSLATION_NOTES, s.SENTIMENT_POLARITY, s.INTENT, s.REQUIRES_REPLY, s.SENTIMENT_CONFIDENCE, s.ANALYSIS_NOTES, s.PROCESSING_SUCCESS, CAST(NULL AS VARCHAR(16777216)) as PROCESSING_ERRORS, s.PROCESSED_AT, s.WORKFLOW_VERSION, CAST(NULL AS TIMESTAMP_NTZ(9)) as CREATED_AT, CAST(NULL AS TIMESTAMP_NTZ(9)) as UPDATED_AT, s.CHANNEL_NAME as BRAND, c.PERMALINK_URL, CAST(NULL AS VARCHAR(16777216)) as THUMBNAIL_URL FROM SOCIAL_MEDIA_DB.ML_FEATURES.COMMENT_SENTIMENT_FEATURES s LEFT JOIN SOCIAL_MEDIA_DB.CORE.DIM_CONTENT c ON s.CONTENT_SK = c.CONTENT_SK UNION ALL SELECT COMMENT_SK, COMMENT_ID, ORIGINAL_TEXT, CASE WHEN PLATFORM = 'musora' THEN 'musora_app' ELSE PLATFORM END as PLATFORM, COMMENT_TIMESTAMP, AUTHOR_NAME, AUTHOR_ID, PARENT_COMMENT_ID, PARENT_COMMENT_TEXT, CONTENT_SK, CONTENT_ID, CONTENT_DESCRIPTION, CHANNEL_SK, CHANNEL_NAME, CHANNEL_DISPLAY_NAME, DETECTED_LANGUAGE, LANGUAGE_CODE, IS_ENGLISH, LANGUAGE_CONFIDENCE, DETECTION_METHOD, HAS_TEXT, TRANSLATED_TEXT, TRANSLATION_PERFORMED, TRANSLATION_CONFIDENCE, TRANSLATION_NOTES, SENTIMENT_POLARITY, INTENT, REQUIRES_REPLY, SENTIMENT_CONFIDENCE, ANALYSIS_NOTES, PROCESSING_SUCCESS, PROCESSING_ERRORS, PROCESSED_AT, WORKFLOW_VERSION, CREATED_AT, UPDATED_AT, CHANNEL_NAME as BRAND, PERMALINK_URL, THUMBNAIL_URL FROM SOCIAL_MEDIA_DB.ML_FEATURES.MUSORA_COMMENT_SENTIMENT_FEATURES",
+        "demographics_query": "SELECT u.id as USER_ID, u.birthday as BIRTHDAY, u.timezone as TIMEZONE, GREATEST(COALESCE(p.difficulty, 0), COALESCE(p.self_report_difficulty, 0), COALESCE(p.method_experience, 0)) AS EXPERIENCE_LEVEL FROM stitch.musora_ecom_db.usora_users u JOIN online_recsys.preprocessed.users p ON u.id = p.user_id"
+    },
+    "demographics": {
+        "age_groups": {
+            "18-24": [18, 24],
+            "25-34": [25, 34],
+            "35-44": [35, 44],
+            "45-54": [45, 54],
+            "55+": [55, 150]
+        },
+        "experience_groups": {
+            "Beginner (0-3)": [0, 3],
+            "Intermediate (4-7)": [4, 7],
+            "Advanced (8-10)": [8, 10]
+        },
+        "top_timezones_count": 15
+    }
+}

visualization/data/data_loader.py

@@ -0,0 +1,427 @@
+"""
+Data loader module for Sentiment Analysis Visualization
+Handles Snowflake connection and data loading with caching
+"""
+import sys
+import os
+import pandas as pd
+import streamlit as st
+from pathlib import Path
+import json
+from datetime import datetime, timedelta
+from dateutil.relativedelta import relativedelta
+
+# Add parent directory to path to import SnowFlakeConnection
+parent_dir = Path(__file__).resolve().parent.parent.parent
+sys.path.append(str(parent_dir))
+
+from visualization.SnowFlakeConnection import SnowFlakeConn
+
+
+class SentimentDataLoader:
+    """
+    Loads sentiment analysis data from Snowflake with caching
+    """
+
+    def __init__(self, config_path=None):
+        """
+        Initialize data loader
+
+        Args:
+            config_path: Path to configuration file
+        """
+        if config_path is None:
+            config_path = Path(__file__).parent.parent / "config" / "viz_config.json"
+
+        with open(config_path, 'r') as f:
+            self.config = json.load(f)
+
+        self.query = self.config['snowflake']['query']
+        self.demographics_query = self.config['snowflake'].get('demographics_query', None)
+
+    @st.cache_data(ttl=300)  # Cache for 5 minutes
+    def load_data(_self, reload=False):
+        """
+        Load sentiment data from Snowflake
+
+        Args:
+            reload: Force reload data (bypass cache)
+
+        Returns:
+            pd.DataFrame: Sentiment analysis data
+        """
+        try:
+            # Connect to Snowflake
+            conn = SnowFlakeConn()
+
+            # Execute query
+            df = conn.run_read_query(_self.query, "sentiment features")
+
+            # Close connection
+            conn.close_connection()
+
+            if df is None or df.empty:
+                st.error("No data returned from Snowflake")
+                return pd.DataFrame()
+
+            # Process dataframe
+            df = _self._process_dataframe(df)
+
+            # Load and merge demographics data for musora_app users
+            if _self.demographics_query:
+                demographics_df = _self.load_demographics_data()
+                df = _self.merge_demographics_with_comments(df, demographics_df)
+
+            return df
+
+        except Exception as e:
+            st.error(f"Error loading data from Snowflake: {e}")
+            return pd.DataFrame()
+
+    def _process_dataframe(self, df):
+        """
+        Process and clean the dataframe
+
+        Args:
+            df: Raw dataframe from Snowflake
+
+        Returns:
+            pd.DataFrame: Processed dataframe
+        """
+        # Convert column names to lowercase (should already be done by run_read_query)
+        df.columns = df.columns.str.lower()
+
+        # Parse datetime columns
+        if 'comment_timestamp' in df.columns:
+            df['comment_timestamp'] = pd.to_datetime(df['comment_timestamp'], errors='coerce')
+
+        if 'processed_at' in df.columns:
+            df['processed_at'] = pd.to_datetime(df['processed_at'], errors='coerce')
+
+        # Handle null values in key columns
+        df['sentiment_polarity'] = df['sentiment_polarity'].fillna('unknown')
+        df['intent'] = df['intent'].fillna('unknown')
+        df['platform'] = df['platform'].fillna('unknown').str.lower()
+        df['brand'] = df['brand'].fillna('unknown').str.lower()
+
+        # Convert requires_reply to boolean
+        if 'requires_reply' in df.columns:
+            df['requires_reply'] = df['requires_reply'].astype(bool)
+
+        # Extract display text (translated if available, otherwise original)
+        df['display_text'] = df.apply(
+            lambda row: row['translated_text'] if pd.notna(row.get('translated_text')) and row.get('is_english') == False
+            else row.get('original_text', ''),
+            axis=1
+        )
+
+        # Create a shortened version for display
+        df['display_text_short'] = df['display_text'].apply(
+            lambda x: x[:100] + '...' if isinstance(x, str) and len(x) > 100 else x
+        )
+
+        return df
+
+    @staticmethod
+    def get_filter_options(df):
+        """
+        Get unique values for filters
+
+        Args:
+            df: Sentiment dataframe
+
+        Returns:
+            dict: Filter options
+        """
+        return {
+            'platforms': sorted(df['platform'].unique().tolist()),
+            'brands': sorted(df['brand'].unique().tolist()),
+            'sentiments': sorted(df['sentiment_polarity'].unique().tolist()),
+            'languages': sorted(df['detected_language'].dropna().unique().tolist()) if 'detected_language' in df.columns else []
+        }
+
+    @staticmethod
+    def apply_filters(df, platforms=None, brands=None, sentiments=None,
+                      date_range=None, languages=None):
+        """
+        Apply filters to dataframe
+
+        Args:
+            df: Sentiment dataframe
+            platforms: List of platforms to include
+            brands: List of brands to include
+            sentiments: List of sentiment polarities to include
+            date_range: Tuple of (start_date, end_date)
+            languages: List of languages to include
+
+        Returns:
+            pd.DataFrame: Filtered dataframe
+        """
+        filtered_df = df.copy()
+
+        if platforms and len(platforms) > 0:
+            filtered_df = filtered_df[filtered_df['platform'].isin(platforms)]
+
+        if brands and len(brands) > 0:
+            filtered_df = filtered_df[filtered_df['brand'].isin(brands)]
+
+        if sentiments and len(sentiments) > 0:
+            filtered_df = filtered_df[filtered_df['sentiment_polarity'].isin(sentiments)]
+
+        if languages and len(languages) > 0:
+            filtered_df = filtered_df[filtered_df['detected_language'].isin(languages)]
+
+        if date_range and len(date_range) == 2 and 'comment_timestamp' in filtered_df.columns:
+            start_date, end_date = date_range
+            filtered_df = filtered_df[
+                (filtered_df['comment_timestamp'] >= pd.Timestamp(start_date)) &
+                (filtered_df['comment_timestamp'] <= pd.Timestamp(end_date))
+            ]
+
+        return filtered_df
+
+    @staticmethod
+    def get_date_range(df, default_days=30):
+        """
+        Get default date range for filtering
+
+        Args:
+            df: Sentiment dataframe
+            default_days: Number of days to include by default
+
+        Returns:
+            tuple: (min_date, max_date)
+        """
+        if 'comment_timestamp' in df.columns and not df.empty:
+            max_date = df['comment_timestamp'].max()
+            min_date = max_date - timedelta(days=default_days)
+            return (min_date, max_date)
+        else:
+            return (datetime.now() - timedelta(days=default_days), datetime.now())
+
+    @st.cache_data(ttl=600)  # Cache for 10 minutes (demographics change less frequently)
+    def load_demographics_data(_self):
+        """
+        Load user demographic data from Snowflake
+
+        Returns:
+            pd.DataFrame: User demographic data with user_id, birthday, timezone, experience_level
+        """
+        if not _self.demographics_query:
+            return pd.DataFrame()
+
+        try:
+            # Connect to Snowflake
+            conn = SnowFlakeConn()
+
+            # Execute demographics query with explicit timestamp conversion
+            # Cast timestamp to string to avoid precision issues, then convert in pandas
+            query_with_cast = _self.demographics_query.replace(
+                "u.birthday as BIRTHDAY",
+                "TO_VARCHAR(u.birthday, 'YYYY-MM-DD HH24:MI:SS.FF6 TZHTZM') as BIRTHDAY"
+            )
+
+            df = conn.run_read_query(query_with_cast, "user demographics")
+
+            # Close connection
+            conn.close_connection()
+
+            if df is None or df.empty:
+                return pd.DataFrame()
+
+            # Process demographics dataframe
+            df = _self._process_demographics_dataframe(df)
+
+            return df
+
+        except Exception as e:
+            # More detailed error message
+            st.warning(f"Could not load demographic data: {str(e)}")
+            import traceback
+            st.error(f"Error reading user demographics: {traceback.format_exc()}")
+            return pd.DataFrame()
+
+    def _process_demographics_dataframe(self, df):
+        """
+        Process and enrich demographic dataframe
+
+        Args:
+            df: Raw demographics dataframe
+
+        Returns:
+            pd.DataFrame: Processed demographics with age and region fields
+        """
+        # Convert column names to lowercase
+        df.columns = df.columns.str.lower()
+
+        # Parse birthday as datetime
+        if 'birthday' in df.columns:
+            # Convert to datetime, handling both string and timestamp formats
+            # First convert to string to ensure consistency
+            df['birthday'] = df['birthday'].astype(str)
+            # Then parse as datetime
+            df['birthday'] = pd.to_datetime(df['birthday'], errors='coerce', utc=True)
+            # Remove timezone info to avoid issues
+            df['birthday'] = df['birthday'].dt.tz_localize(None)
+
+            # Calculate age
+            df['age'] = df['birthday'].apply(self._calculate_age)
+
+            # Create age groups
+            df['age_group'] = df['age'].apply(self._categorize_age)
+
+        # Extract region from timezone
+        if 'timezone' in df.columns:
+            df['timezone_region'] = df['timezone'].apply(self._extract_timezone_region)
+
+        # Create experience level groups
+        if 'experience_level' in df.columns:
+            df['experience_group'] = df['experience_level'].apply(self._categorize_experience)
+
+        # Remove records with null user_id
+        if 'user_id' in df.columns:
+            df = df[df['user_id'].notna()]
+
+        return df
+
+    @staticmethod
+    def _calculate_age(birthday):
+        """
+        Calculate age from birthday
+
+        Args:
+            birthday: datetime object
+
+        Returns:
+            int: Age in years, or None if birthday is invalid
+        """
+        if pd.isna(birthday):
+            return None
+
+        try:
+            today = datetime.now()
+            age = relativedelta(today, birthday).years
+
+            # Sanity check: age should be between 0 and 120
+            if 0 <= age <= 120:
+                return age
+            return None
+        except:
+            return None
+
+    def _categorize_age(self, age):
+        """
+        Categorize age into groups based on config
+
+        Args:
+            age: Age in years
+
+        Returns:
+            str: Age group label or 'Unknown'
+        """
+        if pd.isna(age) or age is None:
+            return 'Unknown'
+
+        age_groups = self.config.get('demographics', {}).get('age_groups', {})
+
+        for group_name, (min_age, max_age) in age_groups.items():
+            if min_age <= age <= max_age:
+                return group_name
+
+        return 'Unknown'
+
+    @staticmethod
+    def _extract_timezone_region(timezone):
+        """
+        Extract region from timezone string (e.g., 'America/New_York' -> 'America')
+
+        Args:
+            timezone: Timezone string
+
+        Returns:
+            str: Region name or 'Unknown'
+        """
+        if pd.isna(timezone) or not isinstance(timezone, str):
+            return 'Unknown'
+
+        # Split by '/' and take the first part
+        parts = timezone.split('/')
+        if len(parts) > 0:
+            return parts[0]
+
+        return 'Unknown'
+
+    def _categorize_experience(self, experience_level):
+        """
+        Categorize experience level into groups based on config
+
+        Args:
+            experience_level: Numeric experience level
+
+        Returns:
+            str: Experience group label or 'Unknown'
+        """
+        if pd.isna(experience_level):
+            return 'Unknown'
+
+        try:
+            exp_level = float(experience_level)
+        except:
+            return 'Unknown'
+
+        exp_groups = self.config.get('demographics', {}).get('experience_groups', {})
+
+        for group_name, (min_exp, max_exp) in exp_groups.items():
+            if min_exp <= exp_level <= max_exp:
+                return group_name
+
+        return 'Unknown'
+
+    def merge_demographics_with_comments(self, comments_df, demographics_df):
+        """
+        Merge demographic data with comment data for musora_app platform only
+
+        Args:
+            comments_df: Main comments dataframe
+            demographics_df: Demographics dataframe
+
+        Returns:
+            pd.DataFrame: Merged dataframe with demographic fields for musora_app comments
+        """
+        if demographics_df.empty:
+            # Add empty demographic columns if no demographics data
+            comments_df['age'] = None
+            comments_df['age_group'] = 'Unknown'
+            comments_df['timezone'] = None
+            comments_df['timezone_region'] = 'Unknown'
+            comments_df['experience_level'] = None
+            comments_df['experience_group'] = 'Unknown'
+            return comments_df
+
+        # Ensure author_id is in the same format for merging
+        if 'author_id' in comments_df.columns and 'user_id' in demographics_df.columns:
+            # Convert both to string for consistent merging
+            comments_df['author_id_str'] = comments_df['author_id'].astype(str)
+            demographics_df['user_id_str'] = demographics_df['user_id'].astype(str)
+
+            # Merge demographics only for musora_app platform
+            merged_df = comments_df.merge(
+                demographics_df[['user_id_str', 'age', 'age_group', 'timezone', 'timezone_region',
+                                'experience_level', 'experience_group']],
+                left_on='author_id_str',
+                right_on='user_id_str',
+                how='left'
+            )
+
+            # Drop temporary merge columns
+            merged_df = merged_df.drop(columns=['author_id_str', 'user_id_str'], errors='ignore')
+
+            # Fill NaN demographic fields with 'Unknown' for non-musora_app platforms
+            demographic_cols = ['age_group', 'timezone_region', 'experience_group']
+            for col in demographic_cols:
+                if col in merged_df.columns:
+                    merged_df[col] = merged_df[col].fillna('Unknown')
+
+            return merged_df
+
+        return comments_df

visualization/requirements.txt

@@ -0,0 +1,17 @@
+# Visualization Requirements
+# Install with: pip install -r requirements.txt
+
+# Core visualization
+streamlit>=1.28.0
+plotly>=5.17.0
+
+# Data processing
+pandas>=2.0.0
+numpy>=1.24.0
+python-dateutil>=2.8.0
+
+# Snowflake connectivity (inherited from parent project)
+snowflake-snowpark-python>=1.8.0
+
+# Environment management (inherited from parent project)
+python-dotenv>=1.0.0

visualization/utils/data_processor.py

@@ -0,0 +1,604 @@
+"""
+Data processing utilities for sentiment analysis
+Handles aggregation, grouping, and transformation operations
+"""
+import pandas as pd
+import numpy as np
+from typing import List, Dict, Tuple
+
+
+class SentimentDataProcessor:
+    """
+    Processes sentiment data for visualization
+    """
+
+    @staticmethod
+    def aggregate_by_dimensions(df, group_by_cols, agg_cols=None):
+        """
+        Aggregate data by specified dimensions
+
+        Args:
+            df: Sentiment dataframe
+            group_by_cols: List of columns to group by
+            agg_cols: Dictionary of columns and aggregation functions
+
+        Returns:
+            pd.DataFrame: Aggregated dataframe
+        """
+        if agg_cols is None:
+            agg_cols = {
+                'comment_sk': 'count',
+                'requires_reply': 'sum'
+            }
+
+        return df.groupby(group_by_cols, as_index=False).agg(agg_cols)
+
+    @staticmethod
+    def get_sentiment_distribution(df, group_by=None):
+        """
+        Calculate sentiment distribution
+
+        Args:
+            df: Sentiment dataframe
+            group_by: Optional column(s) to group by
+
+        Returns:
+            pd.DataFrame: Sentiment distribution
+        """
+        if group_by:
+            # Group by specified columns and sentiment
+            if isinstance(group_by, str):
+                group_by = [group_by]
+
+            sentiment_counts = df.groupby(
+                group_by + ['sentiment_polarity'],
+                as_index=False
+            ).size().rename(columns={'size': 'count'})
+
+            # Calculate percentages within each group
+            sentiment_counts['percentage'] = sentiment_counts.groupby(group_by)['count'].transform(
+                lambda x: (x / x.sum() * 100).round(2)
+            )
+
+        else:
+            # Overall sentiment distribution
+            sentiment_counts = df['sentiment_polarity'].value_counts().reset_index()
+            sentiment_counts.columns = ['sentiment_polarity', 'count']
+            sentiment_counts['percentage'] = (
+                sentiment_counts['count'] / sentiment_counts['count'].sum() * 100
+            ).round(2)
+
+        return sentiment_counts
+
+    @staticmethod
+    def get_intent_distribution(df, group_by=None):
+        """
+        Calculate intent distribution (handles multi-label)
+
+        Args:
+            df: Sentiment dataframe
+            group_by: Optional column(s) to group by
+
+        Returns:
+            pd.DataFrame: Intent distribution
+        """
+        # Explode intents (split comma-separated values)
+        df_exploded = df.copy()
+        df_exploded['intent'] = df_exploded['intent'].str.split(',')
+        df_exploded = df_exploded.explode('intent')
+        df_exploded['intent'] = df_exploded['intent'].str.strip()
+
+        if group_by:
+            # Group by specified columns and intent
+            if isinstance(group_by, str):
+                group_by = [group_by]
+
+            intent_counts = df_exploded.groupby(
+                group_by + ['intent'],
+                as_index=False
+            ).size().rename(columns={'size': 'count'})
+
+            # Calculate percentages within each group
+            intent_counts['percentage'] = intent_counts.groupby(group_by)['count'].transform(
+                lambda x: (x / x.sum() * 100).round(2)
+            )
+
+        else:
+            # Overall intent distribution
+            intent_counts = df_exploded['intent'].value_counts().reset_index()
+            intent_counts.columns = ['intent', 'count']
+            intent_counts['percentage'] = (
+                intent_counts['count'] / intent_counts['count'].sum() * 100
+            ).round(2)
+
+        return intent_counts
+
+    @staticmethod
+    def get_content_summary(df):
+        """
+        Get summary statistics for each content
+
+        Args:
+            df: Sentiment dataframe
+
+        Returns:
+            pd.DataFrame: Content summary with statistics
+        """
+        # Group by content (dropna=False to include records with NULL permalink_url, e.g., YouTube)
+        content_summary = df.groupby(['content_sk', 'content_description', 'permalink_url'], dropna=False).agg({
+            'comment_sk': 'count',
+            'requires_reply': 'sum',
+            'sentiment_polarity': lambda x: x.mode()[0] if len(x.mode()) > 0 else 'unknown'
+        }).reset_index()
+
+        content_summary.columns = [
+            'content_sk', 'content_description', 'permalink_url',
+            'total_comments', 'reply_required_count', 'dominant_sentiment'
+        ]
+
+        # Calculate negative sentiment percentage for each content
+        negative_sentiments = ['negative', 'very_negative']
+        content_negative = df[df['sentiment_polarity'].isin(negative_sentiments)].groupby(
+            'content_sk'
+        ).size().reset_index(name='negative_count')
+
+        content_summary = content_summary.merge(content_negative, on='content_sk', how='left')
+        content_summary['negative_count'] = content_summary['negative_count'].fillna(0)
+        content_summary['negative_percentage'] = (
+            content_summary['negative_count'] / content_summary['total_comments'] * 100
+        ).round(2)
+
+        # Calculate severity score (balances percentage and volume)
+        # Formula: negative_percentage * sqrt(total_comments)
+        # This gives weight to both high negative % and high comment volume
+        content_summary['severity_score'] = (
+            content_summary['negative_percentage'] *
+            (content_summary['total_comments'] ** 0.5)
+        ).round(2)
+
+        return content_summary
+
+    @staticmethod
+    def get_top_poor_sentiment_contents(df, top_n=10, min_comments=1, sort_by='severity_score'):
+        """
+        Get contents with highest poor sentiment based on selected criteria
+
+        Args:
+            df: Sentiment dataframe
+            top_n: Number of top contents to return
+            min_comments: Minimum number of comments a content must have to be included
+            sort_by: Sorting criteria - 'severity_score', 'negative_percentage', 'negative_count', 'total_comments'
+
+        Returns:
+            pd.DataFrame: Top contents with poor sentiment
+        """
+        content_summary = SentimentDataProcessor.get_content_summary(df)
+
+        # Filter by minimum comments
+        content_summary = content_summary[content_summary['total_comments'] >= min_comments]
+
+        # Determine sort columns based on sort_by parameter
+        if sort_by == 'severity_score':
+            # Sort by severity score (balanced), then by negative percentage as tie-breaker
+            sort_columns = ['severity_score', 'negative_percentage']
+        elif sort_by == 'negative_percentage':
+            # Sort by negative percentage, then by total comments
+            sort_columns = ['negative_percentage', 'total_comments']
+        elif sort_by == 'negative_count':
+            # Sort by absolute negative count, then by negative percentage
+            sort_columns = ['negative_count', 'negative_percentage']
+        elif sort_by == 'total_comments':
+            # Sort by total comments volume
+            sort_columns = ['total_comments', 'negative_count']
+        else:
+            # Default to severity score
+            sort_columns = ['severity_score', 'negative_percentage']
+
+        # Sort and get top N
+        top_poor = content_summary.sort_values(
+            by=sort_columns,
+            ascending=[False, False]
+        ).head(top_n)
+
+        return top_poor
+
+    @staticmethod
+    def get_comments_requiring_reply(df):
+        """
+        Get all comments that require reply
+
+        Args:
+            df: Sentiment dataframe
+
+        Returns:
+            pd.DataFrame: Comments requiring reply
+        """
+        reply_df = df[df['requires_reply'] == True].copy()
+
+        # Sort by timestamp (most recent first)
+        if 'comment_timestamp' in reply_df.columns:
+            reply_df = reply_df.sort_values('comment_timestamp', ascending=False)
+
+        return reply_df
+
+    @staticmethod
+    def get_platform_brand_summary(df):
+        """
+        Get summary statistics by platform and brand
+
+        Args:
+            df: Sentiment dataframe
+
+        Returns:
+            pd.DataFrame: Platform and brand summary
+        """
+        summary = df.groupby(['platform', 'brand']).agg({
+            'comment_sk': 'count',
+            'requires_reply': 'sum'
+        }).reset_index()
+
+        summary.columns = ['platform', 'brand', 'total_comments', 'reply_required']
+
+        # Add sentiment distribution
+        sentiment_dist = SentimentDataProcessor.get_sentiment_distribution(
+            df, group_by=['platform', 'brand']
+        )
+
+        # Pivot sentiment distribution
+        sentiment_pivot = sentiment_dist.pivot_table(
+            index=['platform', 'brand'],
+            columns='sentiment_polarity',
+            values='count',
+            fill_value=0
+        ).reset_index()
+
+        # Merge with summary
+        summary = summary.merge(sentiment_pivot, on=['platform', 'brand'], how='left')
+
+        return summary
+
+    @staticmethod
+    def get_temporal_trends(df, freq='D'):
+        """
+        Get temporal trends of sentiment over time
+
+        Args:
+            df: Sentiment dataframe
+            freq: Frequency for aggregation ('D'=daily, 'W'=weekly, 'M'=monthly)
+
+        Returns:
+            pd.DataFrame: Temporal sentiment trends
+        """
+        if 'comment_timestamp' not in df.columns:
+            return pd.DataFrame()
+
+        df_temporal = df.copy()
+        df_temporal['date'] = pd.to_datetime(df_temporal['comment_timestamp']).dt.to_period(freq)
+
+        # Aggregate by date and sentiment
+        trends = df_temporal.groupby(['date', 'sentiment_polarity']).size().reset_index(name='count')
+        trends['date'] = trends['date'].dt.to_timestamp()
+
+        return trends
+
+    @staticmethod
+    def calculate_sentiment_score(df):
+        """
+        Calculate weighted sentiment score
+
+        Args:
+            df: Sentiment dataframe
+
+        Returns:
+            float: Average sentiment score (-2 to +2)
+        """
+        sentiment_weights = {
+            'very_negative': -2,
+            'negative': -1,
+            'neutral': 0,
+            'positive': 1,
+            'very_positive': 2
+        }
+
+        df['sentiment_score'] = df['sentiment_polarity'].map(sentiment_weights)
+        return df['sentiment_score'].mean()
+
+    @staticmethod
+    def get_language_distribution(df):
+        """
+        Get distribution of detected languages
+
+        Args:
+            df: Sentiment dataframe
+
+        Returns:
+            pd.DataFrame: Language distribution
+        """
+        if 'detected_language' not in df.columns:
+            return pd.DataFrame()
+
+        lang_dist = df['detected_language'].value_counts().reset_index()
+        lang_dist.columns = ['language', 'count']
+        lang_dist['percentage'] = (lang_dist['count'] / lang_dist['count'].sum() * 100).round(2)
+
+        return lang_dist
+
+    @staticmethod
+    def get_sentiment_filtered_contents(df, selected_sentiments=None, selected_intents=None,
+                                        top_n=10, min_comments=1, sort_by='severity_score'):
+        """
+        Get contents filtered by selected sentiments and intents with dynamic sorting
+
+        Args:
+            df: Sentiment dataframe
+            selected_sentiments: List of sentiments to filter by (filters by dominant sentiment)
+            selected_intents: List of intents to filter by (content must have at least one comment with these intents)
+            top_n: Number of top contents to return
+            min_comments: Minimum number of comments a content must have
+            sort_by: Sorting criteria - 'severity_score', 'sentiment_percentage', 'sentiment_count', 'total_comments'
+
+        Returns:
+            pd.DataFrame: Filtered and sorted contents
+        """
+        content_summary = SentimentDataProcessor.get_content_summary(df)
+
+        # Filter by minimum comments
+        content_summary = content_summary[content_summary['total_comments'] >= min_comments]
+
+        # If no sentiments selected, default to all sentiments
+        if not selected_sentiments:
+            selected_sentiments = df['sentiment_polarity'].unique().tolist()
+
+        # Filter by dominant sentiment
+        content_summary = content_summary[content_summary['dominant_sentiment'].isin(selected_sentiments)]
+
+        # Filter by intents if specified
+        if selected_intents:
+            # Get content_sks that have at least one comment with the selected intents
+            content_sks_with_intent = set()
+            for intent in selected_intents:
+                matching_contents = df[df['intent'].str.contains(intent, na=False, case=False)]['content_sk'].unique()
+                content_sks_with_intent.update(matching_contents)
+
+            content_summary = content_summary[content_summary['content_sk'].isin(content_sks_with_intent)]
+
+        # Calculate percentage and count for selected sentiments
+        sentiment_counts = df[df['sentiment_polarity'].isin(selected_sentiments)].groupby(
+            'content_sk'
+        ).size().reset_index(name='selected_sentiment_count')
+
+        content_summary = content_summary.merge(sentiment_counts, on='content_sk', how='left')
+        content_summary['selected_sentiment_count'] = content_summary['selected_sentiment_count'].fillna(0)
+        content_summary['selected_sentiment_percentage'] = (
+            content_summary['selected_sentiment_count'] / content_summary['total_comments'] * 100
+        ).round(2)
+
+        # Calculate dynamic severity score based on selected sentiments
+        content_summary['dynamic_severity_score'] = (
+            content_summary['selected_sentiment_percentage'] *
+            (content_summary['total_comments'] ** 0.5)
+        ).round(2)
+
+        # Determine sort columns based on sort_by parameter
+        if sort_by == 'severity_score':
+            sort_columns = ['dynamic_severity_score', 'selected_sentiment_percentage']
+        elif sort_by == 'sentiment_percentage':
+            sort_columns = ['selected_sentiment_percentage', 'total_comments']
+        elif sort_by == 'sentiment_count':
+            sort_columns = ['selected_sentiment_count', 'selected_sentiment_percentage']
+        elif sort_by == 'total_comments':
+            sort_columns = ['total_comments', 'selected_sentiment_count']
+        else:
+            sort_columns = ['dynamic_severity_score', 'selected_sentiment_percentage']
+
+        # Sort and get top N
+        filtered_contents = content_summary.sort_values(
+            by=sort_columns,
+            ascending=[False, False]
+        ).head(top_n)
+
+        return filtered_contents
+
+    @staticmethod
+    def get_demographics_distribution(df, demographic_field, filter_platform='musora_app'):
+        """
+        Get distribution of a demographic field (only for specified platform)
+
+        Args:
+            df: Sentiment dataframe with demographic fields
+            demographic_field: Field to analyze ('age_group', 'timezone', 'timezone_region', 'experience_level', 'experience_group')
+            filter_platform: Platform to filter (default: 'musora_app')
+
+        Returns:
+            pd.DataFrame: Distribution with count and percentage
+        """
+        # Filter for specified platform only
+        if filter_platform and 'platform' in df.columns:
+            df_filtered = df[df['platform'] == filter_platform].copy()
+        else:
+            df_filtered = df.copy()
+
+        if df_filtered.empty or demographic_field not in df_filtered.columns:
+            return pd.DataFrame()
+
+        # Remove 'Unknown' and null values
+        df_filtered = df_filtered[
+            (df_filtered[demographic_field].notna()) &
+            (df_filtered[demographic_field] != 'Unknown')
+        ]
+
+        if df_filtered.empty:
+            return pd.DataFrame()
+
+        # Count distribution
+        distribution = df_filtered[demographic_field].value_counts().reset_index()
+        distribution.columns = [demographic_field, 'count']
+
+        # Calculate percentage
+        distribution['percentage'] = (
+            distribution['count'] / distribution['count'].sum() * 100
+        ).round(2)
+
+        # Sort by count descending
+        distribution = distribution.sort_values('count', ascending=False)
+
+        return distribution
+
+    @staticmethod
+    def get_demographics_by_sentiment(df, demographic_field, filter_platform='musora_app'):
+        """
+        Get sentiment distribution for each demographic group
+
+        Args:
+            df: Sentiment dataframe with demographic fields
+            demographic_field: Field to analyze
+            filter_platform: Platform to filter (default: 'musora_app')
+
+        Returns:
+            pd.DataFrame: Sentiment distribution per demographic group
+        """
+        # Filter for specified platform only
+        if filter_platform and 'platform' in df.columns:
+            df_filtered = df[df['platform'] == filter_platform].copy()
+        else:
+            df_filtered = df.copy()
+
+        if df_filtered.empty or demographic_field not in df_filtered.columns:
+            return pd.DataFrame()
+
+        # Remove 'Unknown' and null values
+        df_filtered = df_filtered[
+            (df_filtered[demographic_field].notna()) &
+            (df_filtered[demographic_field] != 'Unknown')
+        ]
+
+        if df_filtered.empty:
+            return pd.DataFrame()
+
+        # Group by demographic field and sentiment
+        sentiment_by_demo = df_filtered.groupby(
+            [demographic_field, 'sentiment_polarity'],
+            as_index=False
+        ).size().rename(columns={'size': 'count'})
+
+        # Calculate percentage within each demographic group
+        sentiment_by_demo['percentage'] = sentiment_by_demo.groupby(demographic_field)['count'].transform(
+            lambda x: (x / x.sum() * 100).round(2)
+        )
+
+        return sentiment_by_demo
+
+    @staticmethod
+    def get_top_timezones(df, top_n=15, filter_platform='musora_app'):
+        """
+        Get top N timezones with most comments
+
+        Args:
+            df: Sentiment dataframe with timezone field
+            top_n: Number of top timezones to return
+            filter_platform: Platform to filter (default: 'musora_app')
+
+        Returns:
+            pd.DataFrame: Top timezones with counts
+        """
+        return SentimentDataProcessor.get_demographics_distribution(
+            df, 'timezone', filter_platform
+        ).head(top_n)
+
+    @staticmethod
+    def get_timezone_regions_distribution(df, filter_platform='musora_app'):
+        """
+        Get distribution of timezone regions
+
+        Args:
+            df: Sentiment dataframe with timezone_region field
+            filter_platform: Platform to filter (default: 'musora_app')
+
+        Returns:
+            pd.DataFrame: Region distribution with counts
+        """
+        return SentimentDataProcessor.get_demographics_distribution(
+            df, 'timezone_region', filter_platform
+        )
+
+    @staticmethod
+    def get_experience_level_distribution(df, filter_platform='musora_app', use_groups=False):
+        """
+        Get distribution of experience levels
+
+        Args:
+            df: Sentiment dataframe with experience fields
+            filter_platform: Platform to filter (default: 'musora_app')
+            use_groups: If True, use grouped experience levels, otherwise use raw values
+
+        Returns:
+            pd.DataFrame: Experience distribution
+        """
+        field = 'experience_group' if use_groups else 'experience_level'
+        return SentimentDataProcessor.get_demographics_distribution(
+            df, field, filter_platform
+        )
+
+    @staticmethod
+    def get_demographics_summary(df, filter_platform='musora_app'):
+        """
+        Get summary statistics for demographic data
+
+        Args:
+            df: Sentiment dataframe with demographic fields
+            filter_platform: Platform to filter (default: 'musora_app')
+
+        Returns:
+            dict: Summary statistics
+        """
+        # Filter for specified platform only
+        if filter_platform and 'platform' in df.columns:
+            df_filtered = df[df['platform'] == filter_platform].copy()
+        else:
+            df_filtered = df.copy()
+
+        if df_filtered.empty:
+            return {
+                'total_comments': 0,
+                'users_with_demographics': 0,
+                'avg_age': None,
+                'most_common_age_group': 'Unknown',
+                'most_common_region': 'Unknown',
+                'avg_experience': None
+            }
+
+        # Remove records without demographic data
+        df_with_demo = df_filtered[
+            (df_filtered['age'].notna()) |
+            (df_filtered['timezone'].notna()) |
+            (df_filtered['experience_level'].notna())
+        ].copy()
+
+        summary = {
+            'total_comments': len(df_filtered),
+            'users_with_demographics': len(df_with_demo),
+            'coverage_percentage': round(len(df_with_demo) / len(df_filtered) * 100, 2) if len(df_filtered) > 0 else 0
+        }
+
+        # Age statistics
+        if 'age' in df_with_demo.columns:
+            valid_ages = df_with_demo['age'].dropna()
+            summary['avg_age'] = round(valid_ages.mean(), 1) if len(valid_ages) > 0 else None
+
+            age_groups = df_with_demo['age_group'].value_counts()
+            summary['most_common_age_group'] = age_groups.index[0] if len(age_groups) > 0 else 'Unknown'
+
+        # Timezone statistics
+        if 'timezone_region' in df_with_demo.columns:
+            regions = df_with_demo[df_with_demo['timezone_region'] != 'Unknown']['timezone_region'].value_counts()
+            summary['most_common_region'] = regions.index[0] if len(regions) > 0 else 'Unknown'
+
+        # Experience statistics
+        if 'experience_level' in df_with_demo.columns:
+            valid_exp = df_with_demo['experience_level'].dropna()
+            summary['avg_experience'] = round(valid_exp.mean(), 2) if len(valid_exp) > 0 else None
+
+            exp_groups = df_with_demo['experience_group'].value_counts()
+            summary['most_common_experience'] = exp_groups.index[0] if len(exp_groups) > 0 else 'Unknown'
+
+        return summary

visualization/utils/llm_helper.py

@@ -0,0 +1,149 @@
+"""
+LLM Helper for visualization agents
+Handles OpenAI API calls with retry logic and error handling
+"""
+import os
+import json
+from typing import Dict, Any, Optional
+from openai import OpenAI
+from dotenv import load_dotenv
+import time
+
+# Load environment variables from root directory (parent of visualization)
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+VISUALIZATION_DIR = os.path.dirname(SCRIPT_DIR)
+ROOT_DIR = os.path.dirname(VISUALIZATION_DIR)
+load_dotenv(os.path.join(ROOT_DIR, '.env'))
+
+
+class LLMHelper:
+    """
+    Helper class for LLM interactions
+    """
+
+    def __init__(self, model: str = "gpt-5-nano", temperature: float = 1):
+        """
+        Initialize LLM helper
+
+        Args:
+            model: Model name to use
+            temperature: Temperature for generation
+        """
+        self.model = model
+        self.temperature = temperature
+        self.api_key = os.getenv('OPENAI_API_KEY')
+
+        if not self.api_key:
+            raise ValueError("OPENAI_API_KEY not found in environment variables")
+
+        self.client = OpenAI(api_key=self.api_key)
+
+    def get_completion(
+        self,
+        prompt: str,
+        system_message: Optional[str] = None,
+        max_retries: int = 3,
+        json_mode: bool = False
+    ) -> Dict[str, Any]:
+        """
+        Get completion from LLM with retry logic
+
+        Args:
+            prompt: User prompt
+            system_message: Optional system message
+            max_retries: Maximum number of retries
+            json_mode: Whether to force JSON response
+
+        Returns:
+            Dictionary with response data
+        """
+        messages = []
+
+        if system_message:
+            messages.append({"role": "system", "content": system_message})
+
+        messages.append({"role": "user", "content": prompt})
+
+        for attempt in range(max_retries):
+            try:
+                # Prepare API call parameters
+                api_params = {
+                    "model": self.model,
+                    "messages": messages,
+                    "temperature": self.temperature,
+                    "reasoning_effort": "low",
+                    "n": 1
+                }
+
+                # Add response format if JSON mode requested
+                if json_mode:
+                    api_params["response_format"] = {"type": "json_object"}
+
+                # Make API call
+                response = self.client.chat.completions.create(**api_params)
+
+                # Extract response
+                content = response.choices[0].message.content
+
+                # Parse JSON if requested
+                if json_mode:
+                    try:
+                        content = json.loads(content)
+                    except json.JSONDecodeError as e:
+                        return {
+                            'success': False,
+                            'error': f"Failed to parse JSON response: {str(e)}",
+                            'raw_content': content
+                        }
+
+                return {
+                    'success': True,
+                    'content': content,
+                    'model': response.model,
+                    'usage': {
+                        'prompt_tokens': response.usage.prompt_tokens,
+                        'completion_tokens': response.usage.completion_tokens,
+                        'total_tokens': response.usage.total_tokens
+                    }
+                }
+
+            except Exception as e:
+                if attempt < max_retries - 1:
+                    # Wait before retry (exponential backoff)
+                    time.sleep(2 ** attempt)
+                    continue
+                else:
+                    return {
+                        'success': False,
+                        'error': str(e),
+                        'error_type': type(e).__name__
+                    }
+
+        return {
+            'success': False,
+            'error': f"Failed after {max_retries} attempts"
+        }
+
+    def get_structured_completion(
+        self,
+        prompt: str,
+        system_message: str,
+        max_retries: int = 3
+    ) -> Dict[str, Any]:
+        """
+        Get structured JSON completion
+
+        Args:
+            prompt: User prompt
+            system_message: System message
+            max_retries: Maximum retries
+
+        Returns:
+            Structured response dictionary
+        """
+        return self.get_completion(
+            prompt=prompt,
+            system_message=system_message,
+            max_retries=max_retries,
+            json_mode=True
+        )