Deploying sentiment analysis project

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.idea/vcs.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings" defaultProject="true" />
+</project>

Dockerfile

@@ -1,20 +0,0 @@
-FROM python:3.13.5-slim
-
-WORKDIR /app
-
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    curl \
-    git \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY requirements.txt ./
-COPY src/ ./src/
-
-RUN pip3 install -r requirements.txt
-
-EXPOSE 8501
-
-HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health
-
-ENTRYPOINT ["streamlit", "run", "src/streamlit_app.py", "--server.port=8501", "--server.address=0.0.0.0"]

README.md

@@ -1,20 +1,309 @@
+# Musora Sentiment Analysis Dashboard
+
+A Streamlit dashboard for visualising sentiment analysis results from **social media comments** (Facebook, Instagram, YouTube, Twitter) and the **Musora internal app** across brands (Drumeo, Pianote, Guitareo, Singeo, Musora).
+
+---
+
+## Table of Contents
+
+1. [Project Structure](#project-structure)
+2. [How Data Flows](#how-data-flows)
+3. [Data Loading Strategy](#data-loading-strategy)
+4. [Pages](#pages)
+5. [Global Filters & Session State](#global-filters--session-state)
+6. [Snowflake Queries](#snowflake-queries)
+7. [Adding or Changing Things](#adding-or-changing-things)
+8. [Running the App](#running-the-app)
+9. [Configuration Reference](#configuration-reference)
+
+---
+
+## Project Structure
+
+```
+visualization/
+├── app.py                          # Entry point — routing, sidebar, session state
+├── config/
+│   └── viz_config.json             # Colors, query strings, dashboard settings
+├── data/
+│   └── data_loader.py              # All Snowflake queries and caching logic
+├── utils/
+│   ├── data_processor.py           # Pandas aggregations (intent dist, content summary, etc.)
+│   └── metrics.py                  # KPI calculations (sentiment score, urgency, etc.)
+├── components/
+│   ├── dashboard.py                # Dashboard page renderer
+│   ├── sentiment_analysis.py       # Sentiment Analysis page renderer
+│   └── reply_required.py           # Reply Required page renderer
+├── visualizations/
+│   ├── sentiment_charts.py         # Plotly sentiment chart functions
+│   ├── distribution_charts.py      # Plotly distribution / heatmap / scatter functions
+│   ├── demographic_charts.py       # Plotly demographic chart functions
+│   └── content_cards.py            # Streamlit card components (comment cards, content cards)
+├── agents/
+│   └── content_summary_agent.py    # AI analysis agent (OpenAI) for comment summarisation
+├── img/
+│   └── musora.png                  # Sidebar logo
+└── SnowFlakeConnection.py          # Snowflake connection wrapper (Snowpark session)
+```
+
+---
+
+## How Data Flows
+
+```
+Snowflake
+    │
+    ▼
+data_loader.py          ← Three separate loading modes (see below)
+    │
+    ├── load_dashboard_data()   ──► st.session_state['dashboard_df']
+    │                                   └─► app.py sidebar (filter options, counts)
+    │                                   └─► dashboard.py (all charts)
+    │
+    ├── load_sa_data()          ──► st.session_state['sa_contents']
+    │   (on-demand, button)          st.session_state['sa_comments']
+    │                                   └─► sentiment_analysis.py
+    │
+    └── load_reply_required_data() ► st.session_state['rr_df']
+        (on-demand, button)             └─► reply_required.py
+```
+
+**Key principle:** Data is loaded as little as possible, as late as possible.
+
+- The **Dashboard** uses a lightweight query (no text columns, no content join) cached for 24 hours.
+- The **Sentiment Analysis** and **Reply Required** pages never load data automatically — they wait for the user to click **Fetch Data**.
+- All data is stored in `st.session_state` so page navigation and widget interactions do not re-trigger Snowflake queries.
+
+---
+
+## Data Loading Strategy
+
+All loading logic lives in **`data/data_loader.py`** (`SentimentDataLoader` class).
+
+### `load_dashboard_data()`
+- Uses `dashboard_query` from `viz_config.json`.
+- Fetches only: `comment_sk, content_sk, platform, brand, sentiment_polarity, intent, requires_reply, detected_language, comment_timestamp, processed_at, author_id`.
+- No text columns, no `DIM_CONTENT` join — significantly faster than the full query.
+- Also merges demographics data if `demographics_query` is configured.
+- Cached for **24 hours** (`@st.cache_data(ttl=86400)`).
+- Called once by `app.py` at startup; result stored in `st.session_state['dashboard_df']`.
+
+### `load_sa_data(platform, brand, top_n, min_comments, sort_by, sentiments, intents, date_range)`
+- Runs **two** sequential Snowflake queries:
+  1. **Content aggregation** — groups by `content_sk`, counts per sentiment, computes severity score, returns top N.
+  2. **Sampled comments** — for the top N `content_sk`s only, fetches up to 50 comments per sentiment group per content (negative, positive, other), using Snowflake `QUALIFY ROW_NUMBER()`. `display_text` is computed in SQL (`CASE WHEN IS_ENGLISH = FALSE AND TRANSLATED_TEXT IS NOT NULL THEN TRANSLATED_TEXT ELSE ORIGINAL_TEXT END`).
+- Returns a tuple `(contents_df, comments_df)`.
+- Cached for **24 hours**.
+- Called only when the user clicks **Fetch Data** on the Sentiment Analysis page.
+
+### `load_reply_required_data(platforms, brands, date_range)`
+- Runs a single query filtering `REQUIRES_REPLY = TRUE`.
+- Dynamically includes/excludes the social media table and musora table based on selected platforms.
+- `display_text` computed in SQL.
+- Cached for **24 hours**.
+- Called only when the user clicks **Fetch Data** on the Reply Required page.
+
+### Important: SQL Column Qualification
+Both the social media table (`COMMENT_SENTIMENT_FEATURES`) and the content dimension table (`DIM_CONTENT`) share column names. Any `WHERE` clause inside a query that joins these two tables **must** use the table alias prefix (e.g. `s.PLATFORM`, `s.COMMENT_TIMESTAMP`, `s.CHANNEL_NAME`) to avoid Snowflake `ambiguous column name` errors. The musora table (`MUSORA_COMMENT_SENTIMENT_FEATURES`) has no joins so unqualified column names are fine there.
+
+---
+
+## Pages
+
+### Dashboard (`components/dashboard.py`)
+
+**Receives:** `filtered_df` — the lightweight dashboard dataframe (after optional global filter applied by `app.py`).
+
+**Does not need:** text, translations, content URLs. All charts work purely on aggregated columns (sentiment_polarity, brand, platform, intent, requires_reply, comment_timestamp).
+
+**Key sections:**
+- Summary stats + health indicator
+- Sentiment distribution (pie + gauge)
+- Sentiment by brand and platform (stacked + percentage bar charts)
+- Intent analysis
+- Brand-Platform heatmap
+- Reply requirements + urgency breakdown
+- Demographics (age, timezone, experience level) — only rendered if `author_id` is present and demographics were merged
+
+**To add a new chart:** create the chart function in `visualizations/` and call it from `render_dashboard()`. The function receives `filtered_df`.
+
+---
+
+### Sentiment Analysis (`components/sentiment_analysis.py`)
+
+**Receives:** `data_loader` instance only (no dataframe).
+
+**Flow:**
+1. Reads `st.session_state['dashboard_df']` for filter option lists (platforms, brands, sentiments, intents).
+2. Pre-populates platform/brand dropdowns from `st.session_state['global_filters']`.
+3. Shows filter controls (platform, brand, sentiment, intent, top_n, min_comments, sort_by).
+4. On **Fetch Data** click: calls `data_loader.load_sa_data(...)` and stores results in `st.session_state['sa_contents']` and `['sa_comments']`.
+5. Renders content cards, per-content sentiment + intent charts, AI analysis buttons, and sampled comment expanders.
+
+**Pagination:** `st.session_state['sentiment_page']` (5 contents per page). Reset on new fetch.
+
+**Comments:** Sampled (up to 50 negative + 50 positive + 50 neutral per content). These are already in memory after the fetch — no extra query is needed when the user expands a comment section.
+
+**AI Analysis:** Uses `ContentSummaryAgent` (see `agents/`). Results cached in `st.session_state['content_summaries']`.
+
 ---
-title: Sentiment Analysis
-emoji: 🚀
-colorFrom: red
-colorTo: red
-sdk: docker
-app_port: 8501
-tags:
-- streamlit
-pinned: false
-short_description: Sentiment Analysis Dashboard for Musora
-license: cc-by-4.0
+
+### Reply Required (`components/reply_required.py`)
+
+**Receives:** `data_loader` instance only.
+
+**Flow:**
+1. Reads `st.session_state['dashboard_df']` for filter option lists.
+2. Pre-populates platform, brand, and date from `st.session_state['global_filters']`.
+3. On **Fetch Data** click: calls `data_loader.load_reply_required_data(...)` and stores result in `st.session_state['rr_df']`.
+4. Shows urgency breakdown, in-page view filters (priority, platform, brand, intent — applied in Python, no new query), paginated comment cards, and a "Reply by Content" summary.
+
+**Pagination:** `st.session_state['reply_page']` (10 comments per page). Reset on new fetch.
+
+---
+
+## Global Filters & Session State
+
+Global filters live in the sidebar (`app.py`) and are stored in `st.session_state['global_filters']` as a dict:
+
+```python
+{
+    'platforms':  ['facebook', 'instagram'],   # list or []
+    'brands':     ['drumeo'],
+    'sentiments': [],
+    'date_range': (date(2025, 1, 1), date(2025, 12, 31)),  # or None
+}
+```
+
+- **Dashboard:** `app.py` applies global filters to `dashboard_df` using `data_loader.apply_filters()` and passes the result to `render_dashboard()`.
+- **Sentiment Analysis / Reply Required:** global filters are used to pre-populate their own filter widgets. The actual Snowflake query uses those values when the user clicks Fetch. The pages do **not** receive a pre-filtered dataframe.
+
+### Full session state key reference
+
+| Key | Set by | Used by |
+|-----|--------|---------|
+| `dashboard_df` | `app.py` on startup | sidebar (filter options), dashboard, SA + RR (filter option lists) |
+| `global_filters` | sidebar "Apply Filters" button | app.py (dashboard filter), SA + RR (pre-populate widgets) |
+| `filters_applied` | sidebar buttons | app.py (whether to apply filters) |
+| `sa_contents` | SA fetch button | SA page rendering |
+| `sa_comments` | SA fetch button | SA page rendering |
+| `sa_fetch_key` | SA fetch button | SA page (detect stale data) |
+| `rr_df` | RR fetch button | RR page rendering |
+| `rr_fetch_key` | RR fetch button | RR page (detect stale data) |
+| `sentiment_page` | SA page / fetch | SA pagination |
+| `reply_page` | RR page / fetch | RR pagination |
+| `content_summaries` | AI analysis buttons | SA AI analysis display |
+
+---
+
+## Snowflake Queries
+
+All query strings are either stored in `config/viz_config.json` (static queries) or built dynamically in `data/data_loader.py` (page-specific queries).
+
+### Static queries (in `viz_config.json`)
+
+| Key | Purpose |
+|-----|---------|
+| `query` | Full query with all columns (legacy, kept for compatibility) |
+| `dashboard_query` | Lightweight query — no text, no DIM_CONTENT join |
+| `demographics_query` | Joins `usora_users` with `preprocessed.users` to get age/timezone/experience |
+
+### Dynamic queries (built in `data_loader.py`)
+
+| Method | Description |
+|--------|-------------|
+| `_build_sa_content_query()` | Content aggregation for SA page; filters by platform + brand + date |
+| `_build_sa_comments_query()` | Sampled comments for SA page; uses `QUALIFY ROW_NUMBER() <= 50` |
+| `_build_rr_query()` | Reply-required comments; filters by platform/brand/date; conditionally includes social media and/or musora table |
+
+### Data source tables
+
+| Table | Platform | Notes |
+|-------|----------|-------|
+| `SOCIAL_MEDIA_DB.ML_FEATURES.COMMENT_SENTIMENT_FEATURES` | facebook, instagram, youtube, twitter | Needs `LEFT JOIN SOCIAL_MEDIA_DB.CORE.DIM_CONTENT` for `PERMALINK_URL` |
+| `SOCIAL_MEDIA_DB.ML_FEATURES.MUSORA_COMMENT_SENTIMENT_FEATURES` | musora_app | Has `PERMALINK_URL` and `THUMBNAIL_URL` natively; platform stored as `'musora'`, mapped to `'musora_app'` in queries |
+
+---
+
+## Adding or Changing Things
+
+### Add a new chart to the Dashboard
+1. Write the chart function in the appropriate `visualizations/` file.
+2. Call it from `render_dashboard()` in `components/dashboard.py`, passing `filtered_df`.
+3. The chart function receives a lightweight df — it has no text columns but has all the columns listed in `dashboard_query`.
+
+### Add a new filter to the Dashboard sidebar
+1. Add the widget in `app.py` under the "Global Filters" section.
+2. Store the selected value in the `global_filters` dict under `st.session_state`.
+3. Pass it to `data_loader.apply_filters()`.
+
+### Change what the Sentiment Analysis page queries
+- Edit `_build_sa_content_query()` and/or `_build_sa_comments_query()` in `data_loader.py`.
+- If you add new columns to the content aggregation result, also update `_process_sa_content_stats()` so they are available in `contents_df`.
+- If you add new columns to the comments result, update `_process_sa_comments()`.
+
+### Change what the Reply Required page queries
+- Edit `_build_rr_query()` in `data_loader.py`.
+- Remember: all column references inside the social media block (which has a `JOIN`) must be prefixed with `s.` to avoid Snowflake ambiguity errors.
+
+### Change the cache duration
+- `@st.cache_data(ttl=86400)` is set on `load_dashboard_data`, `_fetch_sa_data`, `_fetch_rr_data`, and `load_demographics_data`.
+- Change `86400` (seconds) to the desired TTL, or set `ttl=None` for no expiry.
+- Users can always force a refresh with the "Reload Data" button in the sidebar (which calls `st.cache_data.clear()` and deletes `st.session_state['dashboard_df']`).
+
+### Add a new page
+1. Create `components/new_page.py` with a `render_new_page(data_loader)` function.
+2. Import and add a radio option in `app.py`.
+3. If the page needs its own Snowflake data, add a `load_new_page_data()` method to `SentimentDataLoader` following the same pattern as `load_sa_data`.
+
+### Add a new column to the Dashboard query
+- Edit `dashboard_query` in `config/viz_config.json`.
+- Both UNION branches must select the same columns in the same order.
+- `_process_dashboard_dataframe()` in `data_loader.py` handles basic type casting — add processing there if needed.
+
+---
+
+## Running the App
+
+```bash
+# From the project root
+streamlit run visualization/app.py
+```
+
+**Required environment variables** (in `.env` at project root):
+
+```
+SNOWFLAKE_USER
+SNOWFLAKE_PASSWORD
+SNOWFLAKE_ACCOUNT
+SNOWFLAKE_ROLE
+SNOWFLAKE_DATABASE
+SNOWFLAKE_WAREHOUSE
+SNOWFLAKE_SCHEMA
+```
+
 ---
 
-# Welcome to Streamlit!
+## Configuration Reference
 
-Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
+`config/viz_config.json` controls:
 
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
+| Section | What it configures |
+|---------|-------------------|
+| `color_schemes.sentiment_polarity` | Hex colors for each sentiment level |
+| `color_schemes.intent` | Hex colors for each intent label |
+| `color_schemes.platform` | Hex colors for each platform |
+| `color_schemes.brand` | Hex colors for each brand |
+| `sentiment_order` | Display order for sentiment categories in charts |
+| `intent_order` | Display order for intent categories |
+| `negative_sentiments` | Which sentiment values count as "negative" |
+| `dashboard.default_date_range_days` | Default date filter window (days) |
+| `dashboard.max_comments_display` | Max comments shown per pagination page |
+| `dashboard.chart_height` | Default Plotly chart height |
+| `dashboard.top_n_contents` | Default top-N for content ranking |
+| `snowflake.query` | Full query (legacy, all columns) |
+| `snowflake.dashboard_query` | Lightweight dashboard query (no text columns) |
+| `snowflake.demographics_query` | Demographics join query |
+| `demographics.age_groups` | Age bucket definitions (label → [min, max]) |
+| `demographics.experience_groups` | Experience bucket definitions |
+| `demographics.top_timezones_count` | How many timezones to show in the geographic chart |

processing_brand_sentiment/ARCHITECTURE_PROPOSAL.md

@@ -0,0 +1,437 @@
+# Brand Sentiment Analysis - Architecture Redesign Proposal
+
+## Executive Summary
+
+This document proposes a redesigned multi-agent architecture to address accuracy issues identified during manual evaluation. The new design separates **fact extraction** from **analysis**, adds strict validation, and improves content preprocessing.
+
+---
+
+## Current Issues Analysis
+
+| Issue | Root Cause | Impact |
+|-------|------------|--------|
+| **B8X/B8 variation** | Word-boundary matching misses aliases | Missing relevant posts |
+| **Competitor products attributed to Sabian** | LLM lacks competitor awareness, no strict list enforcement | False positives, wrong product attribution |
+| **Short text language detection** | Lingua fails on short brand-heavy text | Skipping valid English posts |
+| **False positive relevance** | Single-pass relevance + no verification | Pizza oven marked as Sabian discussion |
+| **Long posts with overlapping content** | Poor quote separation, raw thread context | Confusing LLM, extraction from wrong content |
+
+---
+
+## Proposed Architecture
+
+### Design Principles
+
+1. **Separation of Concerns**: Fact extraction vs. interpretation/analysis
+2. **Strict Validation**: Enforce predefined value lists at every step
+3. **Structured Data Flow**: Each agent receives clean, relevant input
+4. **Fail-Safe Defaults**: Conservative approach - when uncertain, mark as not relevant
+
+### New Workflow
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                    1. CONTENT PREPROCESSOR                       │
+│                         (No LLM)                                 │
+│  • Enhanced HTML parsing (better quote separation)               │
+│  • Text cleaning and normalization                               │
+│  • Language detection (skip for short texts < 50 chars)          │
+│  • Keyword screening with aliases (B8 → B8X)                     │
+│  • Extract: cleaned_content, quoted_content, raw_thread_context  │
+└─────────────────────────────┬───────────────────────────────────┘
+                              │
+                              ▼
+              ┌───────────────────────────────┐
+              │  Has any Sabian-related       │
+              │  keywords (primary/contextual)?│
+              └───────────────┬───────────────┘
+                    │                   │
+                   YES                  NO
+                    │                   │
+                    ▼                   ▼
+┌─────────────────────────────────┐    ┌──────────────────┐
+│  2. RELEVANCE & EXTRACTION      │    │  Mark as         │
+│         AGENT (LLM #1)          │    │  NOT RELEVANT    │
+│                                 │    │  (0 LLM calls)   │
+│  INPUT:                         │    └──────────────────┘
+│  • cleaned_content              │
+│  • quoted_content               │
+│  • raw_thread_context           │
+│  • keywords_found               │
+│                                 │
+│  OUTPUT:                        │
+│  • IS_RELEVANT: boolean         │
+│  • RELEVANCE_CONFIDENCE: h/m/l  │
+│  • RELEVANCE_REASON: string     │
+│  • PRODUCTS_MENTIONED: []       │  ← STRICT: only from predefined list
+│  • SABIAN_MENTION_CONTEXT       │
+│  • AUTHOR_ROLE                  │
+│  • COMPETITORS_MENTIONED: []    │  ← Brand names only, no products
+│  • THREAD_CONTEXT_SUMMARY       │  ← 1-2 sentence summary
+└─────────────────┬───────────────┘
+                  │
+                  ▼
+        ┌─────────────────┐
+        │  IS_RELEVANT?   │
+        └────────┬────────┘
+              │        │
+             YES       NO
+              │        │
+              ▼        ▼
+┌─────────────────────────────────┐    ┌──────────────────┐
+│  3. SENTIMENT & INTENT          │    │  Store with      │
+│       ANALYZER (LLM #2)         │    │  is_relevant=F   │
+│                                 │    │  (1 LLM call)    │
+│  INPUT (structured):            │    └──────────────────┘
+│  • cleaned_content              │
+│  • PRODUCTS_MENTIONED           │  ← Pre-validated list
+│  • SABIAN_MENTION_CONTEXT       │
+│  • AUTHOR_ROLE                  │
+│  • COMPETITORS_MENTIONED        │
+│  • THREAD_CONTEXT_SUMMARY       │  ← Clean, concise context
+│                                 │
+│  OUTPUT:                        │
+│  • SENTIMENT_LEVEL              │
+│  • EMOTION_TYPE                 │
+│  • SENTIMENT_CONFIDENCE         │
+│  • SARCASM_DETECTED             │
+│  • PRODUCT_ATTRIBUTES: []       │
+│  • COMPETITOR_PRODUCTS_OWNED: []│
+│  • COMPARISON_TYPE              │
+│  • INTENTS: []                  │
+│  • PURCHASE_STAGE               │
+│  • DECISION_DRIVERS: []         │
+│  • PAIN_POINTS: []              │
+│  • DELIGHT_FACTORS: []          │
+│  • ANALYSIS_NOTES               │
+└─────────────────┬───────────────┘
+                  │
+                  ▼
+┌─────────────────────────────────┐
+│  4. OUTPUT VALIDATOR            │
+│       (No LLM - Rule-based)     │
+│                                 │
+│  • Verify all values from lists │
+│  • Check logical consistency    │
+│  • Flag anomalies for review    │
+│  • Set processing_status        │
+└─────────────────────────────────┘
+```
+
+---
+
+## API Call Summary
+
+| Scenario | Current Calls | New Calls | Notes |
+|----------|--------------|-----------|-------|
+| No keywords found | 0 | 0 | Same |
+| Primary keywords, relevant | 1 | 2 | +1 for better extraction |
+| Primary keywords, not relevant | 1 | 1 | Extraction determines not relevant |
+| Ambiguous keywords, relevant | 2 | 2 | Same |
+| Ambiguous keywords, not relevant | 2 | 1 | Early exit after extraction |
+
+**Net Impact**: Slight increase for some cases, but significantly better accuracy.
+
+---
+
+## Agent Specifications
+
+### Agent 1: Content Preprocessor (No LLM)
+
+**File**: `workflow/agents/content_preprocessor_agent.py`
+
+**Improvements over current**:
+1. Enhanced HTML parsing with better quote/reply separation
+2. Product alias mapping (B8 → B8X, etc.)
+3. Skip language detection for texts < 50 characters
+4. Always process if primary Sabian keywords found (regardless of language detection)
+
+**Product Aliases** (add to brand_config.json):
+```json
+"product_aliases": {
+    "B8": "B8X",
+    "sbrs": "SBR",
+    "hand hammered": "HH",
+    "hand-hammered": "HH"
+}
+```
+
+---
+
+### Agent 2: Relevance & Extraction Agent (LLM #1)
+
+**File**: `workflow/agents/relevance_extraction_agent.py`
+
+**Purpose**: Determine relevance with HIGH confidence and extract verifiable facts.
+
+**Key Design Decisions**:
+
+1. **Strict Product Matching**:
+   - Provide explicit product list in prompt
+   - Instruction: "ONLY return products that EXACTLY match items in this list"
+   - Return empty list if no exact matches (not hallucinated guesses)
+
+2. **Competitor Awareness**:
+   - List competitor BRAND names (not products)
+   - Instruction: "Products like '2002', 'Signature', 'K Custom' belong to competitors, NOT Sabian"
+   - Prevent cross-brand attribution
+
+3. **Thread Context Summarization**:
+   - Summarize in 1-2 sentences maximum
+   - Focus only on information relevant to understanding the post's context
+
+4. **Conservative Relevance**:
+   - When uncertain, mark as NOT relevant
+   - Require explicit Sabian product/brand mention IN THE POST CONTENT
+   - Quoted content mentioning Sabian does NOT make post relevant
+
+**System Prompt Structure**:
+```
+You are a brand mention extractor for Sabian cymbals. Your job is to:
+1. Determine if the POST CONTENT discusses Sabian products
+2. Extract ONLY facts, not interpretations
+
+## CRITICAL RULES
+
+### Rule 1: Relevance Based on POST CONTENT Only
+- The post is relevant ONLY if the POST CONTENT itself mentions Sabian products
+- Quoted/parent content mentioning Sabian does NOT make the post relevant
+- Generic replies ("Thanks!", "Got it!") are NEVER relevant
+
+### Rule 2: Strict Product Matching
+SABIAN PRODUCTS (use ONLY these exact values):
+[HHX, HH, AAX, AA, Artisan, FRX, Omni, Chopper, Stratus, XSR, B8X, SBR]
+
+- Return ONLY products from this list
+- If you see a product not in this list, do NOT include it
+- "2002", "Signature", "Sound Edge", "Formula 602" are PAISTE products, NOT Sabian
+- "K Custom", "A Custom", "K Zildjian" are ZILDJIAN products, NOT Sabian
+- When uncertain, return empty list []
+
+### Rule 3: Competitor Brand Awareness
+COMPETITOR BRANDS: [Zildjian, Paiste, Meinl, Dream Cymbals, Istanbul Agop, Bosphorus]
+
+- Only return competitor BRAND names in competitors_mentioned
+- Do NOT guess competitor products
+
+### Rule 4: Thread Context Summary
+- Summarize thread context in 1-2 sentences maximum
+- Focus on what helps understand the post's topic
+- If thread is about pizza ovens, say "Thread discusses pizza ovens and cooking"
+
+## OUTPUT FORMAT
+Return ONLY valid JSON:
+{
+    "is_relevant": boolean,
+    "relevance_confidence": "high" | "medium" | "low",
+    "relevance_reason": "1-2 sentences explaining decision",
+    "products_mentioned": [],  // ONLY from Sabian list above
+    "sabian_mention_context": "primary_focus" | "significant_mention" | "casual_mention" | "comparison_context" | null,
+    "author_role": "current_owner" | "past_owner" | "potential_buyer" | "never_owned" | "unknown",
+    "competitors_mentioned": [],  // Brand names only
+    "thread_context_summary": "1-2 sentence summary"
+}
+```
+
+---
+
+### Agent 3: Sentiment & Intent Analyzer (LLM #2)
+
+**File**: `workflow/agents/sentiment_analyzer_agent.py`
+
+**Purpose**: Deep analysis on VERIFIED relevant posts with STRUCTURED input.
+
+**Key Design Decisions**:
+
+1. **Receives Pre-Validated Input**:
+   - Products already extracted and validated
+   - Thread context already summarized
+   - Author role already determined
+
+2. **Focused Analysis**:
+   - Sentiment TOWARDS SABIAN ONLY
+   - Intent classification
+   - Pain points / Delights (author's own experience only)
+   - Purchase journey (author's own journey only)
+
+3. **No Hallucination on Products**:
+   - Products are GIVEN in input, not re-extracted
+   - Can only discuss attributes of provided products
+
+**System Prompt Structure**:
+```
+You are a sentiment analyst for Sabian cymbal discussions.
+
+## INPUT CONTEXT (Pre-validated, trust these values)
+- Products mentioned: {products_mentioned}
+- Sabian mention context: {sabian_mention_context}
+- Author role: {author_role}
+- Thread summary: {thread_context_summary}
+- Competitors mentioned: {competitors_mentioned}
+
+## YOUR TASK
+Analyze the sentiment, emotions, and intents in this post about Sabian.
+
+## CRITICAL RULES
+
+### Rule 1: Sabian-Specific Sentiment
+- Sentiment MUST be about Sabian, NOT overall post tone
+- Example: "Love my new kit! The SBR cymbals sound terrible."
+  - Overall: positive | Sabian sentiment: NEGATIVE
+
+### Rule 2: Author Perspective Only
+These fields are ONLY for author's OWN experience:
+- purchase_stage, decision_drivers, pain_points, delight_factors
+- If author is giving ADVICE to others, these should be null/empty
+
+### Rule 3: Use Only Valid Values
+[List all valid values for each field]
+
+## OUTPUT FORMAT
+{
+    "sentiment_level": "...",
+    "emotion_type": "..." or null,
+    "sentiment_confidence": "high" | "medium" | "low",
+    "sarcasm_detected": boolean,
+    "product_attributes": [],
+    "competitor_products_owned": [],
+    "comparison_type": "..." or null,
+    "intents": [],
+    "purchase_stage": "..." or null,
+    "decision_drivers": [],
+    "pain_points": [],
+    "delight_factors": [],
+    "analysis_notes": "1-2 sentences"
+}
+```
+
+---
+
+### Agent 4: Output Validator (No LLM)
+
+**File**: `workflow/agents/output_validator_agent.py`
+
+**Purpose**: Final validation and anomaly detection.
+
+**Validation Rules**:
+
+1. **List Validation**:
+   - All products_mentioned are in Sabian product list
+   - All competitors_mentioned are in competitor list
+   - All categorical values are from predefined lists
+
+2. **Logical Consistency**:
+   - If is_relevant=True, products_mentioned should not be empty (flag if empty)
+   - If sabian_mention_context="primary_focus", products_mentioned should have items
+   - If sentiment_level="very_negative", pain_points should not be empty (warn)
+
+3. **Anomaly Flagging**:
+   - Flag for manual review if inconsistencies detected
+   - Add `validation_flags` field to output
+
+---
+
+## Configuration Changes
+
+### brand_config.json Updates
+
+```json
+{
+    "brand": {
+        "name": "Sabian",
+        "products": ["HHX", "HH", "AAX", "AA", "Artisan", "FRX", "Omni", "Chopper", "Stratus", "XSR", "B8X", "SBR"],
+        "product_aliases": {
+            "B8": "B8X",
+            "sbrs": "SBR",
+            "hhx's": "HHX",
+            "aax's": "AAX"
+        },
+        "competitor_products_warning": [
+            "2002", "Signature", "Sound Edge", "Formula 602", "Giant Beat",
+            "K Custom", "A Custom", "K Zildjian", "A Zildjian", "S Family",
+            "Byzance", "Pure Alloy", "HCS",
+            "Bliss", "Contact", "Energy"
+        ],
+        "competitors": [...]
+    },
+    "preprocessing": {
+        "min_length_for_language_detection": 50,
+        "always_process_if_primary_keyword": true
+    }
+}
+```
+
+---
+
+## File Structure
+
+```
+processing_brand_sentiment/
+├── config_files/
+│   ├── brand_config.json           # Updated with aliases, warnings
+│   ├── workflow_config.json        # Agent configurations
+│   └── analysis_categories.json    # Category definitions (unchanged)
+├── workflow/
+│   ├── orchestrator.py             # Updated workflow graph
+│   └── agents/
+│       ├── base_agent.py           # Base class (unchanged)
+│       ├── content_preprocessor_agent.py    # Enhanced preprocessing
+│       ├── relevance_extraction_agent.py    # NEW: Extraction + relevance
+│       ├── sentiment_analyzer_agent.py      # NEW: Focused analysis
+│       └── output_validator_agent.py        # NEW: Validation
+```
+
+---
+
+## Migration Path
+
+### Phase 1: Configuration Updates
+1. Update brand_config.json with product aliases
+2. Add competitor product warnings
+3. Update preprocessing settings
+
+### Phase 2: New Agents
+1. Create relevance_extraction_agent.py
+2. Create sentiment_analyzer_agent.py
+3. Create output_validator_agent.py
+4. Update content_preprocessor_agent.py
+
+### Phase 3: Orchestrator Update
+1. Update workflow graph with new flow
+2. Update state definition
+3. Add new routing logic
+
+### Phase 4: Testing & Validation
+1. Run on test batch with known issues
+2. Compare accuracy metrics
+3. Fine-tune prompts based on results
+
+---
+
+## Expected Improvements
+
+| Issue | Current Behavior | Expected After |
+|-------|------------------|----------------|
+| B8/B8X | Missed | Caught via alias mapping |
+| Paiste products as Sabian | Attributed to Sabian | Correctly identified as competitor |
+| Short text language | Marked as Latin | Processed as English |
+| False positive (pizza) | Marked relevant | Marked not relevant |
+| Long confusing context | Raw text confuses LLM | Summarized 1-2 sentences |
+
+---
+
+## Success Metrics
+
+1. **Relevance Accuracy**: >99% (currently ~90%)
+2. **Product Attribution Accuracy**: >99% (currently ~85%)
+3. **Sentiment Accuracy**: >95% (current unknown)
+4. **False Positive Rate**: <1%
+5. **False Negative Rate**: <1%
+
+---
+
+## Questions for Review
+
+1. Should we add a manual review queue for flagged posts?
+2. Should thread_context_summary be stored in output for debugging?
+3. Preferred batch size for re-processing existing data?

processing_brand_sentiment/README.md

@@ -0,0 +1,402 @@
+# Brand Sentiment Analysis Pipeline
+
+A modular, scalable system for analyzing forum discussions and social media comments about specific brands using an agentic workflow with LLMs. The initial implementation focuses on **Sabian** (a cymbal manufacturer), but the architecture supports easy addition of new brands through configuration.
+
+## Overview
+
+The pipeline fetches data from Snowflake (forum posts and/or social media comments), preprocesses them (parsing HTML for forums or cleaning plain text for comments), detects language, validates brand relevance, performs comprehensive sentiment and intelligence extraction using OpenAI's API, and stores enriched results back to Snowflake.
+
+## Data Sources
+
+| Source | Table | Output Table | Description |
+|--------|-------|--------------|-------------|
+| **Forums** | `SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_FORUM_POSTS` | `SABIAN_BRAND_ANALYSIS` | Forum posts with thread context |
+| **Comments** | `SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS` | `SABIAN_BRAND_ANALYSIS_COMMENTS` | Social media comments with content context |
+
+## Architecture v4.0
+
+The system uses a 4-agent pipeline that separates **fact extraction** from **analysis** for improved accuracy. Both data sources share the same extraction, analysis, and validation agents - only the preprocessor differs.
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│            1a. CONTENT PREPROCESSOR (Forums)                    │
+│                       (No LLM)                                  │
+│  - HTML parsing with quote/reply separation                     │
+│  - Product alias mapping (B8 → B8X)                             │
+│  - Smart language detection                                     │
+│  - Keyword-based relevance screening                            │
+├─────────────────────────────────────────────────────────────────┤
+│            1b. COMMENT PREPROCESSOR (Comments)                  │
+│                       (No LLM)                                  │
+│  - Plain text cleaning (no HTML)                                │
+│  - Product alias mapping (B8 → B8X)                             │
+│  - Smart language detection                                     │
+│  - Keyword-based relevance screening                            │
+│  - Context: content title + description + parent comment        │
+└─────────────────────────────┬───────────────────────────────────┘
+                              │
+                              ▼
+              ┌───────────────────────────────┐
+              │  Has Sabian-related keywords? │
+              └───────────────┬───────────────┘
+                    │                   │
+                   YES                  NO
+                    │                   │
+                    ▼                   ▼
+┌─────────────────────────────────┐    ┌──────────────────┐
+│  2. RELEVANCE & EXTRACTION      │    │  Mark as         │
+│         AGENT (LLM #1)          │    │  NOT RELEVANT    │
+│         [SHARED]                │    │  (0 LLM calls)   │
+│  - Validates relevance          │    └──────────────────┘
+│  - Extracts products (strict)   │
+│  - Identifies author role       │
+│  - Summarizes context           │
+│  - Detects competitors          │
+└─────────────────┬───────────────┘
+                  │
+                  ▼
+        ┌─────────────────┐
+        │  IS_RELEVANT?   │
+        └────────┬────────┘
+              │        │
+             YES       NO
+              │        │
+              ▼        ▼
+┌─────────────────────────────────┐    ┌──────────────────┐
+│  3. SENTIMENT & INTENT          │    │  Store with      │
+│       ANALYZER (LLM #2)         │    │  is_relevant=F   │
+│       [SHARED]                  │    │  (1 LLM call)    │
+│  - Sabian-specific sentiment    │    └──────────────────┘
+│  - Intent classification        │
+│  - Pain points / Delights       │
+���  - Purchase journey (author)    │
+│  - Competitor products owned    │
+└─────────────────┬───────────────┘
+                  │
+                  ▼
+┌─────────────────────────────────┐
+│  4. OUTPUT VALIDATOR            │
+│       (No LLM - Rule-based)     │
+│       [SHARED]                  │
+│  - Validates all values         │
+│  - Checks logical consistency   │
+│  - Flags anomalies for review   │
+└─────────────────────────────────┘
+```
+
+## Features
+
+- **Multi-Source Support**: Process forums, social media comments, or both
+- **4-Agent Pipeline**: Separation of extraction and analysis for improved accuracy
+- **Strict Product Matching**: Only returns products from predefined list, preventing hallucination
+- **Competitor Awareness**: Knows which products belong to competitors
+- **Smart Language Detection**: Skips detection for short texts, always processes if primary keywords found
+- **Product Alias Mapping**: Handles variations (B8 → B8X, "hand hammered" → HH)
+- **Thread/Comment Context**: LLM summarizes context for clarity
+- **Validation & Anomaly Detection**: Rule-based validator catches errors and flags edge cases
+- **Author Perspective Tracking**: Distinguishes author's own experience from advice to others
+- **Platform Tracking**: Records source platform for each processed item
+
+## Project Structure
+
+```
+processing_brand_sentiment/
+├── config_files/
+│   ├── brand_config.json           # Brand products, aliases, competitors, keywords, data sources
+│   ├── workflow_config.json        # LLM settings, batch sizes, output config (forums + comments)
+│   └── analysis_categories.json    # Sentiment, intent, pain point categories
+├── database/
+│   ├── __init__.py
+│   ├── snowflake_connection.py     # Snowflake connection handler
+│   └── sql/
+│       ├── fetch_forum_posts.sql   # Query for forum posts with thread context
+│       ├── fetch_comments.sql      # Query for social media comments with content context
+│       ├── create_output_table.sql # Forum output schema with views
+│       ├── init_output_table.sql   # Forum table initialization
+│       ├── create_comments_output_table.sql  # Comment output schema with views
+│       └── init_comments_output_table.sql    # Comment table initialization
+├── workflow/
+│   ├── __init__.py
+│   ├── orchestrator.py             # Forum LangGraph workflow coordinator
+│   ├── comment_orchestrator.py     # Comment LangGraph workflow coordinator
+│   └── agents/
+│       ├── __init__.py
+│       ├── base_agent.py           # Abstract base class
+│       ├── content_preprocessor_agent.py    # Forum: HTML parsing, alias mapping
+│       ├── comment_preprocessor_agent.py    # Comments: plain text, comment context
+│       ├── sabian_relevance_extraction_agent.py  # Shared: relevance + extraction
+│       ├── sabian_sentiment_analyzer_agent.py    # Shared: sentiment analysis
+│       └── output_validator_agent.py             # Shared: rule-based validation
+├── utils/
+│   ├── __init__.py
+│   └── html_parser.py              # HTML content extraction (forums only)
+├── logs/                           # Processing logs (auto-created)
+├── main.py                         # Main execution script (multi-source)
+├── .env                            # Environment variables
+└── README.md                       # This file
+```
+
+## Setup
+
+### 1. Install Dependencies
+
+```bash
+pip install langchain-openai langgraph snowflake-snowpark-python python-dotenv pandas beautifulsoup4 lingua-language-detector
+```
+
+### 2. Configure Environment Variables
+
+Ensure `.env` file contains:
+
+```env
+# Snowflake
+SNOWFLAKE_USER=your_user
+SNOWFLAKE_PASSWORD=your_password
+SNOWFLAKE_ACCOUNT=your_account
+SNOWFLAKE_ROLE=your_role
+SNOWFLAKE_DATABASE=SOCIAL_MEDIA_DB
+SNOWFLAKE_WAREHOUSE=your_warehouse
+SNOWFLAKE_SCHEMA=ML_FEATURES
+
+# OpenAI
+OPENAI_API_KEY=your_openai_key
+```
+
+### 3. Initialize Snowflake Tables
+
+Run the initialization scripts before first processing:
+
+```sql
+-- For forums
+database/sql/init_output_table.sql
+
+-- For social media comments
+database/sql/init_comments_output_table.sql
+```
+
+## Usage
+
+### Process All Sources (Default)
+
+```bash
+python main.py
+```
+
+### Process Forums Only
+
+```bash
+python main.py --data-source forums
+```
+
+### Process Social Media Comments Only
+
+```bash
+python main.py --data-source comments
+```
+
+### Process Limited Number
+
+```bash
+python main.py --limit 100
+python main.py --data-source comments --limit 50
+```
+
+### Sequential Processing (Debug Mode)
+
+```bash
+python main.py --limit 50 --sequential
+```
+
+### First Run (Overwrite Mode)
+
+```bash
+python main.py --overwrite --limit 100
+```
+
+### Command-Line Arguments
+
+| Argument | Description | Default |
+|----------|-------------|---------|
+| `--limit N` | Process only N items per source | All unprocessed |
+| `--overwrite` | Overwrite existing table | Append mode |
+| `--sequential` | Single-threaded processing | Parallel |
+| `--config-dir PATH` | Custom config directory | config_files/ |
+| `--data-source SOURCE` | Source to process: `forums`, `comments`, `all` | `all` |
+
+## Configuration
+
+### brand_config.json
+
+Key sections:
+
+```json
+{
+  "brand": {
+    "name": "Sabian",
+    "products": ["HHX", "HH", "AAX", "AA", "Artisan", "FRX", "Omni", "Chopper", "Stratus", "XSR", "B8X", "SBR"],
+    "product_aliases": {
+      "b8": "B8X",
+      "hand hammered": "HH"
+    },
+    "competitor_products_warning": {
+      "paiste_products": ["2002", "signature", "sound edge", "formula 602"],
+      "zildjian_products": ["k custom", "a custom", "k zildjian"]
+    },
+    "competitors": [...]
+  },
+  "data_sources": {
+    "forums": {
+      "table": "SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_FORUM_POSTS",
+      "platform": "musora_forums"
+    },
+    "comments": {
+      "table": "SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS",
+      "platform_column": "PLATFORM"
+    }
+  }
+}
+```
+
+### analysis_categories.json
+
+Defines valid values for all categorical fields:
+
+- `author_role`: current_owner, past_owner, potential_buyer, never_owned, unknown
+- `sabian_mention_context`: primary_focus, significant_mention, casual_mention, comparison_context
+- `sentiment_level`: very_negative, negative, neutral, positive, very_positive
+- `intents`: seeking_information, providing_information, sharing_experience, comparing, praising, criticizing, buying_selling, general_discussion
+- `feedback_aspects`: sound_quality, price_value, durability, playability, versatility, customer_service, availability, aesthetics
+
+## Output Tables
+
+### Forum Output: `SABIAN_BRAND_ANALYSIS`
+
+| Category | Key Columns |
+|----------|-------------|
+| **Identifiers** | POST_ID, THREAD_ID, POST_AUTHOR_ID, PLATFORM |
+| **Content** | ORIGINAL_CONTENT, CLEANED_CONTENT, QUOTED_CONTENT, THREAD_CONTEXT_SUMMARY |
+| **Thread** | THREAD_TITLE, THREAD_FIRST_POST, POST_CREATED_AT, THREAD_STARTED_AT |
+| **Category** | CATEGORY_TITLE, CATEGORY_TOPIC |
+
+### Comment Output: `SABIAN_BRAND_ANALYSIS_COMMENTS`
+
+| Category | Key Columns |
+|----------|-------------|
+| **Identifiers** | COMMENT_SK, COMMENT_ID, PLATFORM, AUTHOR_NAME, AUTHOR_ID |
+| **Content** | ORIGINAL_TEXT, COMMENT_TIMESTAMP |
+| **Context** | CONTENT_SK, CONTENT_ID, CONTENT_DESCRIPTION, PARENT_COMMENT_ID, PARENT_COMMENT_TEXT |
+| **Channel** | CHANNEL_SK, CHANNEL_NAME, CHANNEL_DISPLAY_NAME |
+
+### Shared Analysis Columns (Both Tables)
+
+| Category | Fields | Notes |
+|----------|--------|-------|
+| **Language** | DETECTED_LANGUAGE, LANGUAGE_CODE, IS_ENGLISH | Language detection |
+| **Relevance** | IS_RELEVANT, RELEVANCE_CONFIDENCE, RELEVANCE_REASON | Brand relevance |
+| **Extraction** | PRODUCTS_MENTIONED, AUTHOR_ROLE, SABIAN_MENTION_CONTEXT | From Agent 1 |
+| **Sentiment** | SENTIMENT_LEVEL, EMOTION_TYPE, SENTIMENT_CONFIDENCE | Sabian-specific |
+| **Intents** | INTENTS (multi-label) | What author is trying to accomplish |
+| **Journey** | PURCHASE_STAGE, DECISION_DRIVERS | Author perspective only |
+| **Feedback** | PAIN_POINTS, DELIGHT_FACTORS | Author's own experience |
+| **Competitive** | COMPETITORS_MENTIONED, COMPETITOR_PRODUCTS_OWNED, COMPARISON_TYPE | Competitive intel |
+| **Validation** | VALIDATION_FLAGS, PROCESSING_STATUS | Anomaly detection |
+
+### Processing Status Values
+
+| Status | Description |
+|--------|-------------|
+| `completed` | Successfully processed, no issues |
+| `completed_with_flags` | Processed but has anomalies to review |
+| `validation_failed` | Validation errors detected |
+| `workflow_error` | Unexpected error during processing |
+
+### Available Views
+
+#### Forum Views
+
+| View | Description |
+|------|-------------|
+| `VW_SABIAN_RELEVANT_ANALYSIS` | Only relevant, successfully processed posts |
+| `VW_SABIAN_FLAGGED_POSTS` | Posts with validation flags for review |
+| `VW_SABIAN_SENTIMENT_DISTRIBUTION` | Sentiment breakdown statistics |
+| `VW_SABIAN_PRODUCT_MENTIONS` | Product mention summary |
+| `VW_SABIAN_COMPETITOR_ANALYSIS` | Competitor comparison analysis |
+| `VW_SABIAN_PAIN_POINTS` | Pain point frequency analysis |
+| `VW_SABIAN_AUTHOR_ROLES` | Author role distribution |
+| `VW_SABIAN_COMPETITOR_OWNERSHIP` | Competitor brands owned by authors |
+| `VW_SABIAN_VALIDATION_SUMMARY` | Processing status breakdown |
+
+#### Comment Views
+
+| View | Description |
+|------|-------------|
+| `VW_SABIAN_COMMENTS_RELEVANT_ANALYSIS` | Relevant, successful comments |
+| `VW_SABIAN_COMMENTS_FLAGGED` | Comments with validation flags |
+| `VW_SABIAN_COMMENTS_SENTIMENT_DISTRIBUTION` | Sentiment by platform |
+| `VW_SABIAN_COMMENTS_PRODUCT_MENTIONS` | Product mentions by platform |
+| `VW_SABIAN_COMMENTS_VALIDATION_SUMMARY` | Processing status by platform |
+
+## API Call Efficiency
+
+| Scenario | LLM Calls | Notes |
+|----------|-----------|-------|
+| No keywords found | 0 | Early exit in preprocessor |
+| Primary keywords, relevant | 2 | Extraction + Analysis |
+| Primary keywords, not relevant | 1 | Only Extraction |
+| Non-English content | 0 | Skipped |
+
+## Key Design Decisions
+
+### Why Separate Forum and Comment Preprocessors?
+
+1. **Different input formats**: Forums use HTML (quotes, blockquotes), comments are plain text
+2. **Different context sources**: Forums have thread title + first post + category; comments have content title + description + parent comment
+3. **Shared analysis**: Both feed into the same extraction and analysis agents
+
+### Why Separate Output Tables?
+
+1. **Different identifiers**: Forums use POST_ID/THREAD_ID; comments use COMMENT_SK/COMMENT_ID/PLATFORM
+2. **Different metadata**: Forums have thread context; comments have content/channel metadata
+3. **Clean separation**: Avoids NULL columns and schema confusion
+4. **Shared analysis columns**: All extracted intelligence fields are identical
+
+### Why Platform Column for Forums?
+
+The `PLATFORM` column was added to `SABIAN_BRAND_ANALYSIS` (defaulting to `musora_forums`) to enable cross-source analysis and maintain consistency with the comments table which uses the dynamic platform value from the source data.
+
+## Troubleshooting
+
+### "Table does not exist" on First Run
+
+Run the appropriate init SQL in Snowflake first:
+- Forums: `database/sql/init_output_table.sql`
+- Comments: `database/sql/init_comments_output_table.sql`
+
+### No Comments Being Processed
+
+Check that `SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS` table exists and contains data. The query joins with `DIM_CONTENT` and `DIM_CHANNEL` - verify these dimension tables have matching records.
+
+### Competitor Products Attributed to Sabian
+
+Check `brand_config.json` for `competitor_products_warning` section. Add any missing competitor products.
+
+### API Rate Limits
+
+Use `--sequential` mode or reduce `--limit`:
+```bash
+python main.py --sequential --limit 50
+```
+
+## Schema Version History
+
+| Version | Changes |
+|---------|---------|
+| 1.0 | Initial release |
+| 2.0 | Added author_role, post_type, sabian_mention_context |
+| 3.0 | Removed post_type (merged into intents), unified feedback_aspects |
+| 4.0 | 4-agent pipeline, thread_context_summary, validation flags, product aliases |
+| 4.0+ | Added social media comments support, PLATFORM column, separate comment output table |
+
+## License
+
+Internal use only - Brand sentiment analysis project.

processing_brand_sentiment/config_files/analysis_categories.json

@@ -0,0 +1,123 @@
+{
+    "author_role": {
+        "description": "Author's relationship to Sabian products",
+        "categories": [
+            {"value": "current_owner", "description": "Currently owns/uses Sabian"},
+            {"value": "past_owner", "description": "Previously owned, sold/replaced"},
+            {"value": "potential_buyer", "description": "Considering purchasing Sabian"},
+            {"value": "never_owned", "description": "Explicitly doesn't own Sabian"},
+            {"value": "unknown", "description": "Cannot determine from post"}
+        ]
+    },
+    "sabian_mention_context": {
+        "description": "How prominently Sabian is discussed",
+        "categories": [
+            {"value": "primary_focus", "description": "Sabian is the main topic"},
+            {"value": "significant_mention", "description": "Discussed with detail, not main focus"},
+            {"value": "casual_mention", "description": "Brief mention among other topics"},
+            {"value": "comparison_context", "description": "Mentioned while comparing to competitors"}
+        ]
+    },
+    "sentiment": {
+        "brand_specific": true,
+        "description": "Sentiment TOWARDS SABIAN ONLY (not overall post tone)",
+        "levels": [
+            {"value": "very_negative", "description": "Strong criticism, anger, severe disappointment"},
+            {"value": "negative", "description": "Complaints, dissatisfaction, mild criticism"},
+            {"value": "neutral", "description": "Factual mention, balanced, no clear sentiment"},
+            {"value": "positive", "description": "Satisfaction, appreciation, mild praise"},
+            {"value": "very_positive", "description": "Enthusiasm, strong praise, highly recommend"}
+        ]
+    },
+    "emotions": {
+        "brand_specific": true,
+        "description": "Emotion towards SABIAN specifically",
+        "categories": [
+            {"value": "frustration", "description": "Annoyance with product issues"},
+            {"value": "disappointment", "description": "Unmet expectations"},
+            {"value": "anger", "description": "Strong negative emotion"},
+            {"value": "satisfaction", "description": "Expectations met, content"},
+            {"value": "excitement", "description": "Eagerness, anticipation"},
+            {"value": "curiosity", "description": "Interest, wanting to know more"},
+            {"value": "indifference", "description": "No strong feelings"}
+        ]
+    },
+    "intents": {
+        "multi_label": true,
+        "description": "What the author is trying to accomplish (can select multiple)",
+        "categories": [
+            {"value": "seeking_information", "description": "Asking questions, seeking advice/recommendations"},
+            {"value": "providing_information", "description": "Answering questions, giving advice, helping others"},
+            {"value": "sharing_experience", "description": "Personal experience, review, testimonial, purchase announcement"},
+            {"value": "comparing", "description": "Comparing brands/products against each other"},
+            {"value": "praising", "description": "Actively endorsing, recommending, advocating for Sabian"},
+            {"value": "criticizing", "description": "Actively complaining, warning others, reporting issues"},
+            {"value": "buying_selling", "description": "Listing gear for sale, looking to buy/trade"},
+            {"value": "general_discussion", "description": "General conversation not fitting above"}
+        ]
+    },
+    "purchase_stage": {
+        "author_perspective_only": true,
+        "description": "Author's own purchase journey stage (null if giving advice to others)",
+        "categories": [
+            {"value": "researching", "description": "Gathering info before buying"},
+            {"value": "deciding", "description": "Actively comparing, about to decide"},
+            {"value": "recently_purchased", "description": "Just bought the product"},
+            {"value": "long_term_owner", "description": "Owned for extended period"},
+            {"value": "selling_replacing", "description": "Selling or replacing gear"}
+        ]
+    },
+    "comparison_type": {
+        "description": "Type of competitive comparison (if comparing)",
+        "categories": [
+            {"value": "direct_comparison", "description": "Side-by-side evaluation"},
+            {"value": "preference_statement", "description": "Stating brand preference"},
+            {"value": "switching_to_sabian", "description": "Moving or Moved from competitor to Sabian"},
+            {"value": "switching_from_sabian", "description": "Moving or Moved from Sabian to competitor"}
+        ]
+    },
+    "feedback_aspects": {
+        "description": "Product/brand aspects discussed. Used for BOTH pain_points (negative) and delight_factors (positive)",
+        "categories": [
+            {"value": "sound_quality", "description": "Sound, tone, character, audio qualities"},
+            {"value": "price_value", "description": "Cost, value for money, deals"},
+            {"value": "durability", "description": "Build quality, longevity, cracking/wear"},
+            {"value": "playability", "description": "Feel, response, ease of playing"},
+            {"value": "versatility", "description": "Range of genres/applications, flexibility"},
+            {"value": "customer_service", "description": "Support, warranty, brand interaction"},
+            {"value": "availability", "description": "Stock, ease of finding/purchasing"},
+            {"value": "aesthetics", "description": "Appearance, finish, visual appeal"}
+        ]
+    },
+    "decision_drivers": {
+        "author_perspective_only": true,
+        "description": "What influenced AUTHOR's own purchase decision (empty if giving advice)",
+        "categories": [
+            {"value": "sound_quality", "description": "Sound characteristics"},
+            {"value": "price", "description": "Cost/budget considerations"},
+            {"value": "durability", "description": "Build quality, longevity"},
+            {"value": "artist_endorsement", "description": "Influenced by endorsed artists"},
+            {"value": "peer_recommendation", "description": "Friends/community recommended"},
+            {"value": "hands_on_testing", "description": "Tried before buying"},
+            {"value": "brand_loyalty", "description": "Previous positive experience"},
+            {"value": "versatility", "description": "Multi-genre/application use"},
+            {"value": "online_reviews", "description": "Read reviews that influenced"}
+        ]
+    },
+    "product_attributes": {
+        "description": "Attributes being discussed about Sabian products",
+        "categories": [
+            {"value": "sound_quality", "description": "Tone, character, audio qualities"},
+            {"value": "durability", "description": "Build quality, longevity"},
+            {"value": "price", "description": "Cost and value"},
+            {"value": "playability", "description": "Feel, response"},
+            {"value": "aesthetics", "description": "Appearance, finish"},
+            {"value": "volume", "description": "Loudness, projection"},
+            {"value": "sustain", "description": "How long sound lasts"},
+            {"value": "versatility", "description": "Range of applications"}
+        ]
+    },
+    "analysis_notes_guidelines": {
+        "description": "Keep to 1-2 sentences. Focus on Sabian-specific insights not captured by other fields."
+    }
+}

processing_brand_sentiment/config_files/brand_config.json

@@ -0,0 +1,111 @@
+{
+    "brand": {
+        "name": "Sabian",
+        "description": "Sabian is a Canadian manufacturer of cymbals founded in 1981",
+        "products": [
+            "HHX",
+            "AAX",
+            "Artisan",
+            "FRX",
+            "Omni",
+            "Chopper",
+            "Stratus",
+            "XSR",
+            "B8X",
+            "SBR"
+        ],
+        "product_aliases": {
+            "b8": "B8X",
+            "sbrs": "SBR",
+            "hhxs": "HHX",
+            "aaxs": "AAX",
+            "hhx's": "HHX",
+            "aax's": "AAX"
+        },
+        "product_descriptions": {
+            "HHX": "Hand Hammered Xtreme - Professional series with dark, complex tones",
+            "AAX": "Bright, cutting cymbals for modern music",
+            "Artisan": "Premium hand-crafted cymbals with unique character",
+            "FRX": "Frequency Reduced Xtreme - Lower volume cymbals",
+            "Omni": "Multi-purpose cymbals for various playing styles",
+            "Chopper": "Effect cymbals with unique sound",
+            "Stratus": "Dark, complex sounds for jazz and fusion",
+            "XSR": "Entry-level professional cymbals",
+            "B8X": "Bronze entry-level cymbals",
+            "SBR": "Entry-level brass cymbals"
+        },
+        "competitor_products_warning": {
+            "description": "Products that belong to competitors - DO NOT attribute to Sabian",
+            "paiste_products": ["2002", "signature", "sound edge", "formula 602", "giant beat", "pst", "rude", "masters", "traditionals", "twenty", "dark energy"],
+            "zildjian_products": ["k custom", "a custom", "k zildjian", "a zildjian", "s family", "i family", "l80", "kerope", "constantinople", "k sweet"],
+            "meinl_products": ["byzance", "pure alloy", "hcs", "classics custom", "mb20", "mb10", "soundcaster"],
+            "dream_products": ["bliss", "contact", "energy", "dark matter", "vintage bliss", "eclipse"],
+            "istanbul_products": ["agop", "xist", "traditional", "sultan", "mehmet"]
+        },
+        "competitors": [
+            {
+                "name": "Zildjian",
+                "aliases": ["zildjian", "zil", "z custom", "a custom", "k custom", "k zildjian", "a zildjian"]
+            },
+            {
+                "name": "Meinl",
+                "aliases": ["meinl", "byzance", "classics"]
+            },
+            {
+                "name": "Paiste",
+                "aliases": ["paiste", "2002", "signature", "formula 602", "sound edge"]
+            },
+            {
+                "name": "Dream Cymbals",
+                "aliases": ["dream", "dream cymbals", "bliss"]
+            },
+            {
+                "name": "Istanbul Agop",
+                "aliases": ["istanbul", "agop", "istanbul agop", "istanbul mehmet"]
+            },
+            {
+                "name": "Bosphorus",
+                "aliases": ["bosphorus"]
+            }
+        ]
+    },
+    "relevance_keywords": {
+        "primary": {
+            "description": "Keywords that definitively indicate Sabian content",
+            "keywords": ["sabian", "hhx", "aax", "artisan", "frx", "omni", "chopper", "stratus", "xsr", "b8x", "sbr"]
+        },
+        "contextual": {
+            "description": "Ambiguous keywords that need context verification",
+            "keywords": ["b8"]
+        },
+        "cymbal_context": {
+            "description": "Keywords that provide cymbal-related context for disambiguation",
+            "keywords": ["cymbal", "cymbals", "crash", "ride", "hi-hat", "hihat", "hi hat", "splash", "china", "bell", "stack", "effects"]
+        }
+    },
+    "preprocessing": {
+        "min_length_for_language_detection": 50,
+        "default_language_for_short_text": "English",
+        "always_process_if_primary_keyword": true,
+        "min_content_length": 3
+    },
+    "filter_conditions": {
+        "exclude_access_levels": ["team", "house-coach"],
+        "exclude_post_states": ["deleted", "spam"],
+        "require_content_length_min": 3
+    },
+    "data_sources": {
+        "forums": {
+            "table": "SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_FORUM_POSTS",
+            "description": "Forum posts mentioning Sabian and their products",
+            "sql_query_file": "database/sql/fetch_forum_posts.sql",
+            "platform": "musora_forums"
+        },
+        "comments": {
+            "table": "SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS",
+            "description": "Social media comments potentially related to Sabian brand",
+            "sql_query_file": "database/sql/fetch_comments.sql",
+            "platform_column": "PLATFORM"
+        }
+    }
+}

processing_brand_sentiment/config_files/workflow_config.json

@@ -0,0 +1,60 @@
+{
+    "llm": {
+        "default_model": "gpt-5-nano",
+        "default_temperature": 0.2,
+        "max_retries": 3,
+        "timeout": 60
+    },
+    "agents": {
+        "preprocessor": {
+            "name": "PreprocessorAgent",
+            "description": "Deterministic agent for HTML parsing, text cleaning, language detection",
+            "model": "gpt-5-nano",
+            "temperature": 0.0,
+            "uses_llm": false
+        },
+        "relevance_validator": {
+            "name": "RelevanceValidatorAgent",
+            "description": "Lightweight LLM for disambiguation of ambiguous terms (HH, AA)",
+            "model": "gpt-5-nano",
+            "temperature": 0.0,
+            "max_retries": 2
+        },
+        "brand_analyzer": {
+            "name": "SabianAnalyzerAgent",
+            "description": "Comprehensive brand analysis for Sabian products",
+            "model": "gpt-5-nano",
+            "temperature": 0.2,
+            "max_retries": 3
+        }
+    },
+    "workflow": {
+        "parallel_processing": {
+            "enabled": true,
+            "worker_calculation": "CPU count - 2, max 5 workers",
+            "max_workers": 5,
+            "min_batch_size": 20,
+            "max_batch_size": 500
+        },
+        "thread_context": {
+            "enabled": true,
+            "include_thread_title": true,
+            "include_first_post": true
+        }
+    },
+    "output": {
+        "table_name": "SABIAN_BRAND_ANALYSIS",
+        "database": "SOCIAL_MEDIA_DB",
+        "schema": "ML_FEATURES"
+    },
+    "comments_output": {
+        "table_name": "SABIAN_BRAND_ANALYSIS_COMMENTS",
+        "database": "SOCIAL_MEDIA_DB",
+        "schema": "ML_FEATURES"
+    },
+    "logging": {
+        "level": "INFO",
+        "log_directory": "logs",
+        "log_file_prefix": "brand_sentiment_processing"
+    }
+}

processing_brand_sentiment/database/__init__.py

@@ -0,0 +1,8 @@
+"""
+Database module for brand sentiment analysis.
+Contains Snowflake connection handler and SQL query utilities.
+"""
+
+from .snowflake_connection import SnowFlakeConn
+
+__all__ = ['SnowFlakeConn']

processing_brand_sentiment/database/snowflake_connection.py

@@ -0,0 +1,240 @@
+"""
+Snowflake connection handler for brand sentiment analysis.
+Provides methods for reading data, executing queries, and storing results.
+"""
+
+import os
+from snowflake.snowpark import Session
+from dotenv import load_dotenv
+import logging
+import pandas as pd
+from typing import Optional, List, Any
+
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+
+
+class SnowFlakeConn:
+    """
+    Handles Snowflake database connections and operations for brand sentiment analysis.
+    """
+
+    def __init__(self):
+        """Initialize Snowflake connection."""
+        self.session = self.connect_to_snowflake()
+
+    def connect_to_snowflake(self) -> Session:
+        """
+        Create a connection to Snowflake using environment variables.
+
+        Returns:
+            Snowflake Session object
+        """
+        conn = dict(
+            user=self.get_credential("SNOWFLAKE_USER"),
+            password=self.get_credential("SNOWFLAKE_PASSWORD"),
+            account=self.get_credential("SNOWFLAKE_ACCOUNT"),
+            role=self.get_credential("SNOWFLAKE_ROLE"),
+            database=self.get_credential("SNOWFLAKE_DATABASE"),
+            warehouse=self.get_credential("SNOWFLAKE_WAREHOUSE"),
+            schema=self.get_credential("SNOWFLAKE_SCHEMA"),
+        )
+
+        session = Session.builder.configs(conn).create()
+        logger.info("Successfully connected to Snowflake")
+        return session
+
+    def get_credential(self, key: str) -> str:
+        """
+        Get credential from environment variables.
+
+        Args:
+            key: Environment variable name
+
+        Returns:
+            Credential value
+        """
+        return os.getenv(key)
+
+    def run_read_query(self, query: str, description: str = "data") -> pd.DataFrame:
+        """
+        Execute a SQL query that fetches data.
+
+        Args:
+            query: SQL query string
+            description: Description of what data is being fetched
+
+        Returns:
+            Pandas DataFrame containing query results
+        """
+        try:
+            dataframe = self.session.sql(query).to_pandas()
+            dataframe.columns = dataframe.columns.str.lower()
+            logger.info(f"Successfully read {len(dataframe)} rows for {description}")
+            return dataframe
+        except Exception as e:
+            logger.error(f"Error reading {description}: {e}")
+            raise
+
+    def store_df_to_snowflake(
+        self,
+        table_name: str,
+        dataframe: pd.DataFrame,
+        database: str = "SOCIAL_MEDIA_DB",
+        schema: str = "ML_FEATURES",
+        overwrite: bool = False
+    ) -> None:
+        """
+        Store a DataFrame to Snowflake.
+
+        Args:
+            table_name: Target table name
+            dataframe: DataFrame to store
+            database: Target database
+            schema: Target schema
+            overwrite: If True, overwrite existing data; if False, append
+        """
+        try:
+            self.session.use_database(database)
+            self.session.use_schema(schema)
+
+            dataframe = dataframe.reset_index(drop=True)
+            dataframe.columns = dataframe.columns.str.upper()
+
+            self.session.write_pandas(
+                df=dataframe,
+                table_name=table_name.strip().upper(),
+                auto_create_table=True,
+                overwrite=overwrite,
+                use_logical_type=True
+            )
+            logger.info(f"Successfully stored {len(dataframe)} rows to {table_name}")
+
+        except Exception as e:
+            logger.error(f"Error storing data to {table_name}: {e}")
+            raise
+
+    def execute_sql_file(self, file_path: str) -> Optional[List[Any]]:
+        """
+        Execute SQL queries from a file.
+
+        Args:
+            file_path: Path to SQL file
+
+        Returns:
+            Query result or None for DDL/DML
+        """
+        try:
+            with open(file_path, 'r', encoding='utf-8') as file:
+                sql_content = file.read()
+
+            result = self.session.sql(sql_content).collect()
+            logger.info(f"Successfully executed SQL from {file_path}")
+            return result
+        except Exception as e:
+            logger.error(f"Error executing SQL file {file_path}: {e}")
+            return None
+
+    def execute_query(self, query: str, description: str = "query") -> Optional[List[Any]]:
+        """
+        Execute a SQL query and return results.
+
+        Args:
+            query: SQL query string
+            description: Description of the query for logging
+
+        Returns:
+            Query results
+        """
+        try:
+            result = self.session.sql(query).collect()
+            logger.info(f"Successfully executed {description}")
+            return result
+        except Exception as e:
+            logger.error(f"Error executing {description}: {e}")
+            return None
+
+    def fetch_forum_posts_with_context(
+        self,
+        sql_file_path: str,
+        limit: Optional[int] = None
+    ) -> pd.DataFrame:
+        """
+        Fetch forum posts with thread context from SQL file.
+
+        Args:
+            sql_file_path: Path to the SQL query file
+            limit: Optional limit on number of posts to fetch
+
+        Returns:
+            DataFrame containing forum posts with context
+        """
+        try:
+            with open(sql_file_path, 'r', encoding='utf-8') as f:
+                query = f.read()
+
+            # Add limit if specified
+            if limit:
+                # Strip whitespace first, then semicolon, to handle Windows line endings
+                query = query.strip().rstrip(';') + f"\nLIMIT {limit};"
+
+            df = self.run_read_query(query, "forum posts with context")
+
+            # Validate required columns
+            required_cols = ['post_id', 'post_content', 'thread_id']
+            missing_cols = [col for col in required_cols if col not in df.columns]
+            if missing_cols:
+                logger.warning(f"Missing expected columns: {missing_cols}")
+
+            return df
+
+        except Exception as e:
+            logger.error(f"Error fetching forum posts: {e}")
+            raise
+
+    def fetch_comments(
+        self,
+        sql_file_path: str,
+        limit: Optional[int] = None
+    ) -> pd.DataFrame:
+        """
+        Fetch social media comments with context from SQL file.
+
+        Args:
+            sql_file_path: Path to the SQL query file
+            limit: Optional limit on number of comments to fetch
+
+        Returns:
+            DataFrame containing comments with context
+        """
+        try:
+            with open(sql_file_path, 'r', encoding='utf-8') as f:
+                query = f.read()
+
+            # Add limit if specified
+            if limit:
+                query = query.strip().rstrip(';') + f"\nLIMIT {limit};"
+
+            df = self.run_read_query(query, "social media comments with context")
+
+            # Validate required columns
+            required_cols = ['comment_sk', 'comment_id', 'comment_text', 'platform']
+            missing_cols = [col for col in required_cols if col not in df.columns]
+            if missing_cols:
+                logger.warning(f"Missing expected columns: {missing_cols}")
+
+            return df
+
+        except Exception as e:
+            logger.error(f"Error fetching comments: {e}")
+            raise
+
+    def close_connection(self) -> None:
+        """Close the Snowflake session."""
+        try:
+            self.session.close()
+            logger.info("Snowflake connection closed")
+        except Exception as e:
+            logger.error(f"Error closing connection: {e}")

processing_brand_sentiment/database/sql/create_comments_output_table.sql

@@ -0,0 +1,161 @@
+-- Create the output table for Sabian brand sentiment analysis on social media comments
+-- Stores processed comments with extracted brand intelligence
+-- Schema Version 4.0: Same analysis fields as forum table, different source identifiers
+
+CREATE TABLE IF NOT EXISTS SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS (
+    -- Source identifiers (comment-specific)
+    COMMENT_SK NUMBER(38,0),
+    COMMENT_ID VARCHAR(16777216),
+    ORIGINAL_TEXT VARCHAR(16777216),
+    PLATFORM VARCHAR(16777216),
+    COMMENT_TIMESTAMP TIMESTAMP_NTZ(9),
+    AUTHOR_NAME VARCHAR(16777216),
+    AUTHOR_ID VARCHAR(16777216),
+    CONTENT_SK NUMBER(38,0),
+    CONTENT_ID VARCHAR(16777216),
+    CONTENT_DESCRIPTION VARCHAR(16777216),
+    CHANNEL_SK NUMBER(38,0),
+    CHANNEL_NAME VARCHAR(16777216),
+    CHANNEL_DISPLAY_NAME VARCHAR(16777216),
+    PARENT_COMMENT_ID VARCHAR(16777216),
+    PARENT_COMMENT_TEXT VARCHAR(16777216),
+
+    -- Language detection
+    DETECTED_LANGUAGE VARCHAR(100),
+    LANGUAGE_CODE VARCHAR(10),
+    IS_ENGLISH BOOLEAN,
+
+    -- Relevance assessment
+    IS_RELEVANT BOOLEAN,
+    RELEVANCE_CONFIDENCE VARCHAR(20),
+    RELEVANCE_REASON VARCHAR(500),
+
+    -- Author classification
+    AUTHOR_ROLE VARCHAR(50),  -- current_owner, past_owner, potential_buyer, never_owned, unknown
+    SABIAN_MENTION_CONTEXT VARCHAR(50),  -- primary_focus, significant_mention, casual_mention, comparison_context
+
+    -- Sentiment analysis
+    SENTIMENT_LEVEL VARCHAR(20),
+    EMOTION_TYPE VARCHAR(50),
+    SENTIMENT_TARGET VARCHAR(50),
+    SENTIMENT_CONFIDENCE VARCHAR(20),
+
+    -- Product information (stored as JSON arrays)
+    PRODUCTS_MENTIONED VARCHAR(16777216),  -- JSON array: ["HHX", "AAX"]
+    PRODUCT_ATTRIBUTES VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+    PURCHASE_STAGE VARCHAR(50),
+
+    -- Competitive intelligence
+    COMPETITORS_MENTIONED VARCHAR(16777216),  -- JSON array: ["Zildjian", "Meinl"]
+    COMPETITOR_PRODUCTS_OWNED VARCHAR(16777216),  -- JSON array: competitor brands the AUTHOR owns
+    COMPARISON_TYPE VARCHAR(50),
+    COMPETITIVE_POSITIONING VARCHAR(500),
+    BRAND_SWITCHING VARCHAR(100),
+
+    -- Customer journey (AUTHOR PERSPECTIVE ONLY - null if giving advice to others)
+    INTENTS VARCHAR(16777216),  -- JSON array (multi-label): ["sharing_experience", "praising"]
+    DECISION_DRIVERS VARCHAR(16777216),  -- JSON array: ["sound_quality", "price"]
+    PAIN_POINTS VARCHAR(16777216),  -- JSON array: ["price_value", "availability"]
+    DELIGHT_FACTORS VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+
+    -- Analysis notes
+    ANALYSIS_NOTES VARCHAR(16777216),
+    SARCASM_DETECTED BOOLEAN,
+
+    -- Validation results
+    VALIDATION_PASSED BOOLEAN,
+    VALIDATION_ERRORS VARCHAR(16777216),  -- JSON array of error messages
+    VALIDATION_WARNINGS VARCHAR(16777216),  -- JSON array of warning messages
+    VALIDATION_FLAGS VARCHAR(16777216),  -- JSON array of anomaly flags
+
+    -- Processing metadata
+    PROCESSING_SUCCESS BOOLEAN,
+    PROCESSING_STATUS VARCHAR(50),  -- completed, completed_with_flags, validation_failed, workflow_error
+    PROCESSING_ERRORS VARCHAR(16777216),
+    PROCESSED_AT TIMESTAMP_LTZ(9) DEFAULT CURRENT_TIMESTAMP(),
+    WORKFLOW_VERSION VARCHAR(20) DEFAULT '4.0'
+)
+COMMENT = 'Brand sentiment analysis results for Sabian social media comments. Schema v4.0: 4-agent pipeline with extraction/analysis separation and validation.';
+
+-- Create indexes for common query patterns
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_SK ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(COMMENT_SK);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_PLATFORM ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(PLATFORM);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_IS_RELEVANT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(IS_RELEVANT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_SENTIMENT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(SENTIMENT_LEVEL);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_PROCESSED_AT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(PROCESSED_AT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_AUTHOR_ROLE ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(AUTHOR_ROLE);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_MENTION_CTX ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(SABIAN_MENTION_CONTEXT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_COMMENTS_STATUS ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS(PROCESSING_STATUS);
+
+-- Create view for relevant comments only
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMMENTS_RELEVANT_ANALYSIS AS
+SELECT *
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE;
+
+-- Create view for comments needing review (flagged by validator)
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMMENTS_FLAGGED AS
+SELECT
+    COMMENT_SK,
+    COMMENT_ID,
+    PLATFORM,
+    ORIGINAL_TEXT,
+    IS_RELEVANT,
+    RELEVANCE_CONFIDENCE,
+    RELEVANCE_REASON,
+    PRODUCTS_MENTIONED,
+    SABIAN_MENTION_CONTEXT,
+    SENTIMENT_LEVEL,
+    VALIDATION_FLAGS,
+    VALIDATION_WARNINGS,
+    PROCESSING_STATUS
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS
+WHERE PROCESSING_STATUS = 'completed_with_flags'
+   OR VALIDATION_PASSED = FALSE
+ORDER BY PROCESSED_AT DESC;
+
+-- Create view for sentiment distribution
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMMENTS_SENTIMENT_DISTRIBUTION AS
+SELECT
+    PLATFORM,
+    SENTIMENT_LEVEL,
+    EMOTION_TYPE,
+    SENTIMENT_TARGET,
+    COUNT(*) AS COMMENT_COUNT,
+    COUNT(CASE WHEN SARCASM_DETECTED = TRUE THEN 1 END) AS SARCASM_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+GROUP BY PLATFORM, SENTIMENT_LEVEL, EMOTION_TYPE, SENTIMENT_TARGET
+ORDER BY COMMENT_COUNT DESC;
+
+-- Create view for product mentions summary
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMMENTS_PRODUCT_MENTIONS AS
+SELECT
+    PLATFORM,
+    TRIM(product.VALUE::STRING) AS PRODUCT,
+    SENTIMENT_LEVEL,
+    COUNT(*) AS MENTION_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('positive', 'very_positive') THEN 1 END) AS POSITIVE_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('negative', 'very_negative') THEN 1 END) AS NEGATIVE_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS,
+    LATERAL FLATTEN(INPUT => TRY_PARSE_JSON(PRODUCTS_MENTIONED)) AS product
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+  AND PRODUCTS_MENTIONED IS NOT NULL
+GROUP BY PLATFORM, TRIM(product.VALUE::STRING), SENTIMENT_LEVEL
+ORDER BY MENTION_COUNT DESC;
+
+-- Create view for validation summary
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMMENTS_VALIDATION_SUMMARY AS
+SELECT
+    PLATFORM,
+    PROCESSING_STATUS,
+    VALIDATION_PASSED,
+    COUNT(*) AS COMMENT_COUNT,
+    COUNT(CASE WHEN IS_RELEVANT = TRUE THEN 1 END) AS RELEVANT_COUNT,
+    COUNT(CASE WHEN IS_RELEVANT = FALSE THEN 1 END) AS NOT_RELEVANT_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS
+GROUP BY PLATFORM, PROCESSING_STATUS, VALIDATION_PASSED
+ORDER BY COMMENT_COUNT DESC;

processing_brand_sentiment/database/sql/create_output_table.sql

@@ -0,0 +1,250 @@
+-- Create the output table for Sabian brand sentiment analysis
+-- Stores processed forum posts with extracted brand intelligence
+-- Schema Version 4.0: Added THREAD_CONTEXT_SUMMARY, validation fields, and processing status
+
+CREATE TABLE IF NOT EXISTS SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS (
+    -- Source identifiers
+    POST_ID NUMBER(38,0) PRIMARY KEY,
+    THREAD_ID NUMBER(38,0),
+    POST_AUTHOR_ID NUMBER(38,0),
+
+    -- Original and processed content
+    ORIGINAL_CONTENT VARCHAR(16777216),
+    CLEANED_CONTENT VARCHAR(16777216),
+    QUOTED_CONTENT VARCHAR(16777216),
+    THREAD_CONTEXT VARCHAR(16777216),  -- Raw thread context (legacy)
+    THREAD_CONTEXT_SUMMARY VARCHAR(4096),  -- NEW v4.0: Summarized thread context for analysis
+
+    -- Thread metadata
+    THREAD_TITLE VARCHAR(16777216),
+    THREAD_FIRST_POST VARCHAR(16777216),
+
+    -- Timestamps
+    POST_CREATED_AT TIMESTAMP_LTZ(9),
+    THREAD_STARTED_AT TIMESTAMP_LTZ(9),
+
+    -- Category information
+    CATEGORY_TITLE VARCHAR(16777216),
+    CATEGORY_TOPIC VARCHAR(16777216),
+
+    -- Language detection
+    DETECTED_LANGUAGE VARCHAR(100),
+    LANGUAGE_CODE VARCHAR(10),
+    IS_ENGLISH BOOLEAN,
+
+    -- Relevance assessment
+    IS_RELEVANT BOOLEAN,
+    RELEVANCE_CONFIDENCE VARCHAR(20),
+    RELEVANCE_REASON VARCHAR(500),
+
+    -- Author classification
+    AUTHOR_ROLE VARCHAR(50),  -- current_owner, past_owner, potential_buyer, never_owned, unknown
+    SABIAN_MENTION_CONTEXT VARCHAR(50),  -- primary_focus, significant_mention, casual_mention, comparison_context
+
+    -- Sentiment analysis
+    SENTIMENT_LEVEL VARCHAR(20),
+    EMOTION_TYPE VARCHAR(50),
+    SENTIMENT_TARGET VARCHAR(50),
+    SENTIMENT_CONFIDENCE VARCHAR(20),
+
+    -- Product information (stored as JSON arrays)
+    PRODUCTS_MENTIONED VARCHAR(16777216),  -- JSON array: ["HHX", "AAX"]
+    PRODUCT_ATTRIBUTES VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+    PURCHASE_STAGE VARCHAR(50),
+
+    -- Competitive intelligence
+    COMPETITORS_MENTIONED VARCHAR(16777216),  -- JSON array: ["Zildjian", "Meinl"]
+    COMPETITOR_PRODUCTS_OWNED VARCHAR(16777216),  -- JSON array: competitor brands the AUTHOR owns
+    COMPARISON_TYPE VARCHAR(50),
+    COMPETITIVE_POSITIONING VARCHAR(500),
+    BRAND_SWITCHING VARCHAR(100),
+
+    -- Customer journey (AUTHOR PERSPECTIVE ONLY - null if giving advice to others)
+    INTENTS VARCHAR(16777216),  -- JSON array (multi-label): ["sharing_experience", "praising"]
+    DECISION_DRIVERS VARCHAR(16777216),  -- JSON array: ["sound_quality", "price"]
+    PAIN_POINTS VARCHAR(16777216),  -- JSON array: ["price_value", "availability"]
+    DELIGHT_FACTORS VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+
+    -- Analysis notes
+    ANALYSIS_NOTES VARCHAR(16777216),
+    SARCASM_DETECTED BOOLEAN,
+
+    -- Validation results (NEW v4.0)
+    VALIDATION_PASSED BOOLEAN,
+    VALIDATION_ERRORS VARCHAR(16777216),  -- JSON array of error messages
+    VALIDATION_WARNINGS VARCHAR(16777216),  -- JSON array of warning messages
+    VALIDATION_FLAGS VARCHAR(16777216),  -- JSON array of anomaly flags (e.g., "sarcasm_detected", "low_confidence_relevant")
+
+    -- Platform identifier
+    PLATFORM VARCHAR(50) DEFAULT 'musora_forums',
+
+    -- Processing metadata
+    PROCESSING_SUCCESS BOOLEAN,
+    PROCESSING_STATUS VARCHAR(50),  -- NEW v4.0: completed, completed_with_flags, validation_failed, workflow_error
+    PROCESSING_ERRORS VARCHAR(16777216),
+    PROCESSED_AT TIMESTAMP_LTZ(9) DEFAULT CURRENT_TIMESTAMP(),
+    WORKFLOW_VERSION VARCHAR(20) DEFAULT '4.0'
+)
+COMMENT = 'Brand sentiment analysis results for Sabian forum posts. Schema v4.0: 4-agent pipeline with extraction/analysis separation, thread context summarization, and validation.';
+
+-- Create indexes for common query patterns
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_THREAD_ID ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(THREAD_ID);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_IS_RELEVANT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(IS_RELEVANT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_SENTIMENT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(SENTIMENT_LEVEL);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_PROCESSED_AT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(PROCESSED_AT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_AUTHOR_ROLE ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(AUTHOR_ROLE);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_MENTION_CONTEXT ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(SABIAN_MENTION_CONTEXT);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_PROCESSING_STATUS ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(PROCESSING_STATUS);
+CREATE INDEX IF NOT EXISTS IDX_SABIAN_VALIDATION_FLAGS ON SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS(VALIDATION_PASSED);
+
+-- Create view for relevant posts only
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_RELEVANT_ANALYSIS AS
+SELECT *
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE;
+
+-- Create view for posts needing review (flagged by validator)
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_FLAGGED_POSTS AS
+SELECT
+    POST_ID,
+    THREAD_ID,
+    CLEANED_CONTENT,
+    THREAD_CONTEXT_SUMMARY,
+    IS_RELEVANT,
+    RELEVANCE_CONFIDENCE,
+    RELEVANCE_REASON,
+    PRODUCTS_MENTIONED,
+    SABIAN_MENTION_CONTEXT,
+    SENTIMENT_LEVEL,
+    VALIDATION_FLAGS,
+    VALIDATION_WARNINGS,
+    PROCESSING_STATUS
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+WHERE PROCESSING_STATUS = 'completed_with_flags'
+   OR VALIDATION_PASSED = FALSE
+ORDER BY PROCESSED_AT DESC;
+
+-- Create view for sentiment distribution
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_SENTIMENT_DISTRIBUTION AS
+SELECT
+    SENTIMENT_LEVEL,
+    EMOTION_TYPE,
+    SENTIMENT_TARGET,
+    COUNT(*) AS POST_COUNT,
+    COUNT(CASE WHEN SARCASM_DETECTED = TRUE THEN 1 END) AS SARCASM_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+GROUP BY SENTIMENT_LEVEL, EMOTION_TYPE, SENTIMENT_TARGET
+ORDER BY POST_COUNT DESC;
+
+-- Create view for product mentions summary
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_PRODUCT_MENTIONS AS
+SELECT
+    TRIM(product.VALUE::STRING) AS PRODUCT,
+    SENTIMENT_LEVEL,
+    COUNT(*) AS MENTION_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('positive', 'very_positive') THEN 1 END) AS POSITIVE_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('negative', 'very_negative') THEN 1 END) AS NEGATIVE_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS,
+    LATERAL FLATTEN(INPUT => TRY_PARSE_JSON(PRODUCTS_MENTIONED)) AS product
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+  AND PRODUCTS_MENTIONED IS NOT NULL
+GROUP BY TRIM(product.VALUE::STRING), SENTIMENT_LEVEL
+ORDER BY MENTION_COUNT DESC;
+
+-- Create view for competitor analysis
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMPETITOR_ANALYSIS AS
+SELECT
+    TRIM(competitor.VALUE::STRING) AS COMPETITOR,
+    COMPARISON_TYPE,
+    BRAND_SWITCHING,
+    COUNT(*) AS MENTION_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('positive', 'very_positive') THEN 1 END) AS POSITIVE_SENTIMENT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('negative', 'very_negative') THEN 1 END) AS NEGATIVE_SENTIMENT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS,
+    LATERAL FLATTEN(INPUT => TRY_PARSE_JSON(COMPETITORS_MENTIONED)) AS competitor
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+  AND COMPETITORS_MENTIONED IS NOT NULL
+GROUP BY TRIM(competitor.VALUE::STRING), COMPARISON_TYPE, BRAND_SWITCHING
+ORDER BY MENTION_COUNT DESC;
+
+-- Create view for pain points analysis
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_PAIN_POINTS AS
+SELECT
+    TRIM(pain_point.VALUE::STRING) AS PAIN_POINT,
+    COUNT(*) AS OCCURRENCE_COUNT,
+    ARRAY_AGG(DISTINCT SENTIMENT_LEVEL) AS SENTIMENT_LEVELS
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS,
+    LATERAL FLATTEN(INPUT => TRY_PARSE_JSON(PAIN_POINTS)) AS pain_point
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+  AND PAIN_POINTS IS NOT NULL
+GROUP BY TRIM(pain_point.VALUE::STRING)
+ORDER BY OCCURRENCE_COUNT DESC;
+
+-- Create view for author role analysis
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_AUTHOR_ROLES AS
+SELECT
+    AUTHOR_ROLE,
+    SABIAN_MENTION_CONTEXT,
+    COUNT(*) AS POST_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('positive', 'very_positive') THEN 1 END) AS POSITIVE_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('negative', 'very_negative') THEN 1 END) AS NEGATIVE_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL = 'neutral' THEN 1 END) AS NEUTRAL_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+GROUP BY AUTHOR_ROLE, SABIAN_MENTION_CONTEXT
+ORDER BY POST_COUNT DESC;
+
+-- Create view for competitor ownership analysis
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_COMPETITOR_OWNERSHIP AS
+SELECT
+    TRIM(competitor.VALUE::STRING) AS COMPETITOR_OWNED,
+    AUTHOR_ROLE,
+    COUNT(*) AS AUTHOR_COUNT,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('positive', 'very_positive') THEN 1 END) AS POSITIVE_TOWARD_SABIAN,
+    COUNT(CASE WHEN SENTIMENT_LEVEL IN ('negative', 'very_negative') THEN 1 END) AS NEGATIVE_TOWARD_SABIAN
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS,
+    LATERAL FLATTEN(INPUT => TRY_PARSE_JSON(COMPETITOR_PRODUCTS_OWNED)) AS competitor
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+  AND COMPETITOR_PRODUCTS_OWNED IS NOT NULL
+GROUP BY TRIM(competitor.VALUE::STRING), AUTHOR_ROLE
+ORDER BY AUTHOR_COUNT DESC;
+
+-- Create view for mention context by sentiment
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_MENTION_DEPTH AS
+SELECT
+    SABIAN_MENTION_CONTEXT,
+    SENTIMENT_LEVEL,
+    COUNT(*) AS POST_COUNT,
+    AVG(CASE
+        WHEN SENTIMENT_LEVEL = 'very_positive' THEN 2
+        WHEN SENTIMENT_LEVEL = 'positive' THEN 1
+        WHEN SENTIMENT_LEVEL = 'neutral' THEN 0
+        WHEN SENTIMENT_LEVEL = 'negative' THEN -1
+        WHEN SENTIMENT_LEVEL = 'very_negative' THEN -2
+        ELSE 0
+    END) AS AVG_SENTIMENT_SCORE
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+WHERE IS_RELEVANT = TRUE
+  AND PROCESSING_SUCCESS = TRUE
+GROUP BY SABIAN_MENTION_CONTEXT, SENTIMENT_LEVEL
+ORDER BY SABIAN_MENTION_CONTEXT, POST_COUNT DESC;
+
+-- Create view for validation flags analysis (NEW v4.0)
+CREATE OR REPLACE VIEW SOCIAL_MEDIA_DB.ML_FEATURES.VW_SABIAN_VALIDATION_SUMMARY AS
+SELECT
+    PROCESSING_STATUS,
+    VALIDATION_PASSED,
+    COUNT(*) AS POST_COUNT,
+    COUNT(CASE WHEN IS_RELEVANT = TRUE THEN 1 END) AS RELEVANT_COUNT,
+    COUNT(CASE WHEN IS_RELEVANT = FALSE THEN 1 END) AS NOT_RELEVANT_COUNT
+FROM SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS
+GROUP BY PROCESSING_STATUS, VALIDATION_PASSED
+ORDER BY POST_COUNT DESC;

processing_brand_sentiment/database/sql/fetch_comments.sql

@@ -0,0 +1,82 @@
+-- Query to fetch social media comments with context for brand sentiment analysis
+-- Source: SOCIAL_MEDIA_DB.brand_sentiment.SABIAN_comments (same structure as CORE.FACT_COMMENTS)
+-- Includes: comment content, parent comment text, content metadata, channel info
+-- Excludes: official accounts, already-processed comments, empty comments
+
+SELECT
+    -- Comment identifiers
+    fc.COMMENT_SK,
+    fc.COMMENT_ID,
+    fc.PLATFORM,
+    fc.MESSAGE AS COMMENT_TEXT,
+    fc.CREATED_TIME AS COMMENT_TIMESTAMP,
+    fc.AUTHOR_NAME,
+    fc.AUTHOR_ID,
+    fc.LIKE_COUNT,
+    fc.PARENT_COMMENT_ID,
+    fc.REPLIES_COUNT,
+    fc.COMMENT_LENGTH,
+    fc.IS_ACTIVE AS COMMENT_IS_ACTIVE,
+
+    -- Parent comment information (self-join to get parent comment text)
+    parent_fc.MESSAGE AS PARENT_COMMENT_TEXT,
+
+    -- Content information
+    dc.CONTENT_SK,
+    dc.CONTENT_ID,
+    dc.CONTENT_TYPE,
+    dc.MESSAGE AS CONTENT_DESCRIPTION,
+    dc.TITLE AS CONTENT_TITLE,
+    dc.PERMALINK_URL,
+    dc.CREATED_TIME AS CONTENT_TIMESTAMP,
+
+    -- Channel information
+    dch.CHANNEL_SK,
+    dch.CHANNEL_NAME,
+    dch.CHANNEL_DISPLAY_NAME
+
+FROM
+    SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS fc
+
+-- Left join to get parent comment text if it exists
+LEFT JOIN
+    SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_COMMENTS parent_fc
+    ON fc.PARENT_COMMENT_ID = parent_fc.COMMENT_ID
+    AND fc.PLATFORM = parent_fc.PLATFORM
+
+INNER JOIN
+    SOCIAL_MEDIA_DB.CORE.DIM_CONTENT dc
+    ON fc.CONTENT_SK = dc.CONTENT_SK
+
+INNER JOIN
+    SOCIAL_MEDIA_DB.CORE.DIM_CHANNEL dch
+    ON dc.CHANNEL_NAME = dch.CHANNEL_NAME
+    AND dc.PLATFORM = dch.PLATFORM
+
+-- Left join with output table to exclude already-processed comments
+LEFT JOIN
+    SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS sba
+    ON fc.COMMENT_SK = sba.COMMENT_SK
+
+WHERE
+    -- Active records only
+    fc.IS_ACTIVE = TRUE
+    AND dc.IS_ACTIVE = TRUE
+    AND dch.IS_ACTIVE = TRUE
+
+    -- Exclude official accounts
+    AND (fc.AUTHOR_NAME IS NULL OR fc.AUTHOR_NAME NOT IN (
+        'Musora', 'Drumeo', 'Pianote',
+        '@PianoteOfficial', '@DrumeoOfficial', '@MusoraOfficial'
+    ))
+
+    -- Exclude already-processed comments
+    AND sba.COMMENT_SK IS NULL
+
+    -- Ensure comment has content
+    AND fc.MESSAGE IS NOT NULL
+    AND TRIM(fc.MESSAGE) != ''
+    AND LENGTH(TRIM(fc.MESSAGE)) > 0
+
+ORDER BY
+    fc.CREATED_TIME DESC;

processing_brand_sentiment/database/sql/fetch_forum_posts.sql

@@ -0,0 +1,106 @@
+-- Query to fetch forum posts with thread context for brand sentiment analysis
+-- Includes: post content, thread context (title, first post), parent relationships
+-- Excludes: team/house-coach posts, already-processed posts, deleted posts
+
+WITH thread_first_posts AS (
+    -- Get the first post (by creation date) for each thread to use as context
+    -- Using ROW_NUMBER for reliable first post identification
+    SELECT
+        THREAD_ID,
+        POST_CONTENT AS FIRST_POST_CONTENT,
+        POST_AUTHOR_ID AS FIRST_POST_AUTHOR_ID,
+        POST_CREATED_AT AS FIRST_POST_CREATED_AT
+    FROM (
+        SELECT
+            THREAD_ID,
+            POST_CONTENT,
+            POST_AUTHOR_ID,
+            POST_CREATED_AT,
+            ROW_NUMBER() OVER (PARTITION BY THREAD_ID ORDER BY POST_CREATED_AT ASC) AS rn
+        FROM SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_FORUM_POSTS
+        WHERE POST_CONTENT IS NOT NULL
+          AND TRIM(POST_CONTENT) != ''
+    ) ranked
+    WHERE rn = 1
+)
+
+SELECT
+    -- Post identifiers
+    fp.POST_ID,
+    fp.POST_AUTHOR_ID,
+    fp.THREAD_ID,
+
+    -- Post content (may contain HTML with quoted parent)
+    fp.POST_CONTENT,
+
+    -- Post timestamps
+    fp.POST_CREATED_AT,
+    fp.POST_EDITED_ON,
+    fp.POST_PUBLISHED_ON,
+    fp.POST_STATE,
+
+    -- Parent/Child relationships (for context)
+    fp.PROMPTING_POST_ID,
+    fp.PARENT_ID,
+    fp.PARENT_CONTENT,
+    fp.PARENT_AUTHOR_ID,
+    fp.PARENT_CREATED_AT,
+    fp.CHILD_ID,
+    fp.CHILD_CONTENT,
+
+    -- Thread context
+    fp.THREAD_TITLE,
+    fp.THREAD_SLUG,
+    fp.THREAD_STATE,
+    fp.THREAD_LOCKED,
+    fp.THREAD_PINNED,
+    fp.THREAD_POST_COUNT,
+    fp.THREAD_PUBLISHED_ON,
+
+    -- First post of the thread (for context)
+    tfp.FIRST_POST_CONTENT AS THREAD_FIRST_POST,
+    tfp.FIRST_POST_CREATED_AT AS THREAD_STARTED_AT,
+
+    -- Category information
+    fp.CATEGORY_ID,
+    fp.CATEGORY_BRAND,
+    fp.CATEGORY_DESCRIPTION,
+    fp.CATEGORY_TITLE,
+    fp.CATEGORY_TOPIC,
+    fp.CATEGORY_SLUG,
+
+    -- Access levels (for filtering)
+    fp.POST_AUTHOR_ACCESS_LEVEL,
+    fp.PARENT_AUTHOR_ACCESS_LEVEL,
+    fp.CHILD_AUTHOR_ACCESS_LEVEL
+
+FROM
+    SOCIAL_MEDIA_DB.BRAND_SENTIMENT.SABIAN_FORUM_POSTS fp
+
+-- Join to get thread's first post for context
+LEFT JOIN
+    thread_first_posts tfp ON fp.THREAD_ID = tfp.THREAD_ID
+
+-- Left join with output table to exclude already-processed posts
+LEFT JOIN
+    SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS sba
+    ON fp.POST_ID = sba.POST_ID
+
+WHERE
+    -- Exclude team and house-coach posts (internal comments)
+    (fp.POST_AUTHOR_ACCESS_LEVEL IS NULL OR fp.POST_AUTHOR_ACCESS_LEVEL NOT IN ('team', 'house-coach'))
+
+    -- Exclude deleted posts
+    AND (fp.POST_STATE IS NULL OR fp.POST_STATE != 'deleted')
+    AND fp.POST_DELETED_AT IS NULL
+
+    -- Exclude already-processed posts
+    AND sba.POST_ID IS NULL
+
+    -- Ensure post has content
+    AND fp.POST_CONTENT IS NOT NULL
+    AND TRIM(fp.POST_CONTENT) != ''
+    AND LENGTH(TRIM(fp.POST_CONTENT)) > 0
+
+ORDER BY
+    fp.POST_CREATED_AT DESC;

processing_brand_sentiment/database/sql/init_comments_output_table.sql

@@ -0,0 +1,78 @@
+-- Initialize empty output table for Sabian brand sentiment analysis on social media comments
+-- Run this script BEFORE the first processing run to create the table structure
+-- This prevents "table not found" errors when the fetch query tries to check for already-processed comments
+
+CREATE TABLE IF NOT EXISTS SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS_COMMENTS (
+    -- Source identifiers (comment-specific)
+    COMMENT_SK NUMBER(38,0),
+    COMMENT_ID VARCHAR(16777216),
+    ORIGINAL_TEXT VARCHAR(16777216),
+    PLATFORM VARCHAR(16777216),
+    COMMENT_TIMESTAMP TIMESTAMP_NTZ(9),
+    AUTHOR_NAME VARCHAR(16777216),
+    AUTHOR_ID VARCHAR(16777216),
+    CONTENT_SK NUMBER(38,0),
+    CONTENT_ID VARCHAR(16777216),
+    CONTENT_DESCRIPTION VARCHAR(16777216),
+    CHANNEL_SK NUMBER(38,0),
+    CHANNEL_NAME VARCHAR(16777216),
+    CHANNEL_DISPLAY_NAME VARCHAR(16777216),
+    PARENT_COMMENT_ID VARCHAR(16777216),
+    PARENT_COMMENT_TEXT VARCHAR(16777216),
+
+    -- Language detection
+    DETECTED_LANGUAGE VARCHAR(100),
+    LANGUAGE_CODE VARCHAR(10),
+    IS_ENGLISH BOOLEAN,
+
+    -- Relevance assessment
+    IS_RELEVANT BOOLEAN,
+    RELEVANCE_CONFIDENCE VARCHAR(20),
+    RELEVANCE_REASON VARCHAR(500),
+
+    -- Author classification
+    AUTHOR_ROLE VARCHAR(50),  -- current_owner, past_owner, potential_buyer, never_owned, unknown
+    SABIAN_MENTION_CONTEXT VARCHAR(50),  -- primary_focus, significant_mention, casual_mention, comparison_context
+
+    -- Sentiment analysis
+    SENTIMENT_LEVEL VARCHAR(20),
+    EMOTION_TYPE VARCHAR(50),
+    SENTIMENT_TARGET VARCHAR(50),
+    SENTIMENT_CONFIDENCE VARCHAR(20),
+
+    -- Product information (stored as JSON arrays)
+    PRODUCTS_MENTIONED VARCHAR(16777216),  -- JSON array: ["HHX", "AAX"]
+    PRODUCT_ATTRIBUTES VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+    PURCHASE_STAGE VARCHAR(50),
+
+    -- Competitive intelligence
+    COMPETITORS_MENTIONED VARCHAR(16777216),  -- JSON array: ["Zildjian", "Meinl"]
+    COMPETITOR_PRODUCTS_OWNED VARCHAR(16777216),  -- JSON array: competitor brands the AUTHOR owns
+    COMPARISON_TYPE VARCHAR(50),
+    COMPETITIVE_POSITIONING VARCHAR(500),
+    BRAND_SWITCHING VARCHAR(100),
+
+    -- Customer journey (AUTHOR PERSPECTIVE ONLY - null if giving advice to others)
+    INTENTS VARCHAR(16777216),  -- JSON array (multi-label): ["sharing_experience", "praising"]
+    DECISION_DRIVERS VARCHAR(16777216),  -- JSON array: ["sound_quality", "price"]
+    PAIN_POINTS VARCHAR(16777216),  -- JSON array: ["price_value", "availability"]
+    DELIGHT_FACTORS VARCHAR(16777216),  -- JSON array: ["sound_quality", "durability"]
+
+    -- Analysis notes
+    ANALYSIS_NOTES VARCHAR(16777216),
+    SARCASM_DETECTED BOOLEAN,
+
+    -- Validation results
+    VALIDATION_PASSED BOOLEAN,
+    VALIDATION_ERRORS VARCHAR(16777216),  -- JSON array of error messages
+    VALIDATION_WARNINGS VARCHAR(16777216),  -- JSON array of warning messages
+    VALIDATION_FLAGS VARCHAR(16777216),  -- JSON array of anomaly flags
+
+    -- Processing metadata
+    PROCESSING_SUCCESS BOOLEAN,
+    PROCESSING_STATUS VARCHAR(50),  -- completed, completed_with_flags, validation_failed, workflow_error
+    PROCESSING_ERRORS VARCHAR(16777216),
+    PROCESSED_AT TIMESTAMP_LTZ(9) DEFAULT CURRENT_TIMESTAMP(),
+    WORKFLOW_VERSION VARCHAR(20) DEFAULT '4.0'
+)
+COMMENT = 'Brand sentiment analysis results for Sabian social media comments. Schema v4.0: 4-agent pipeline with extraction/analysis separation and validation.';

processing_brand_sentiment/database/sql/init_output_table.sql

@@ -0,0 +1,89 @@
+-- Initialize empty output table for Sabian brand sentiment analysis
+-- Run this script BEFORE the first processing run to create the table structure
+-- This prevents "table not found" errors when the fetch query tries to check for already-processed posts
+-- Schema Version 4.0: Added THREAD_CONTEXT_SUMMARY, validation fields, and processing status
+
+CREATE TABLE IF NOT EXISTS SOCIAL_MEDIA_DB.ML_FEATURES.SABIAN_BRAND_ANALYSIS (
+    -- Source identifiers
+    POST_ID NUMBER(38,0) PRIMARY KEY,
+    THREAD_ID NUMBER(38,0),
+    POST_AUTHOR_ID NUMBER(38,0),
+
+    -- Original and processed content
+    ORIGINAL_CONTENT VARCHAR(16777216),
+    CLEANED_CONTENT VARCHAR(16777216),
+    QUOTED_CONTENT VARCHAR(16777216),
+    THREAD_CONTEXT VARCHAR(16777216),  -- Raw thread context (legacy)
+    THREAD_CONTEXT_SUMMARY VARCHAR(4096),  -- NEW v4.0: Summarized thread context
+
+    -- Thread metadata
+    THREAD_TITLE VARCHAR(16777216),
+    THREAD_FIRST_POST VARCHAR(16777216),
+
+    -- Timestamps
+    POST_CREATED_AT TIMESTAMP_LTZ(9),
+    THREAD_STARTED_AT TIMESTAMP_LTZ(9),
+
+    -- Category information
+    CATEGORY_TITLE VARCHAR(16777216),
+    CATEGORY_TOPIC VARCHAR(16777216),
+
+    -- Language detection
+    DETECTED_LANGUAGE VARCHAR(100),
+    LANGUAGE_CODE VARCHAR(10),
+    IS_ENGLISH BOOLEAN,
+
+    -- Relevance assessment
+    IS_RELEVANT BOOLEAN,
+    RELEVANCE_CONFIDENCE VARCHAR(20),
+    RELEVANCE_REASON VARCHAR(500),
+
+    -- Author classification
+    AUTHOR_ROLE VARCHAR(50),  -- current_owner, past_owner, potential_buyer, never_owned, unknown
+    SABIAN_MENTION_CONTEXT VARCHAR(50),  -- primary_focus, significant_mention, casual_mention, comparison_context
+
+    -- Sentiment analysis
+    SENTIMENT_LEVEL VARCHAR(20),
+    EMOTION_TYPE VARCHAR(50),
+    SENTIMENT_TARGET VARCHAR(50),
+    SENTIMENT_CONFIDENCE VARCHAR(20),
+
+    -- Product information (stored as JSON arrays)
+    PRODUCTS_MENTIONED VARCHAR(16777216),
+    PRODUCT_ATTRIBUTES VARCHAR(16777216),
+
+    -- Competitive intelligence
+    COMPETITORS_MENTIONED VARCHAR(16777216),
+    COMPETITOR_PRODUCTS_OWNED VARCHAR(16777216),  -- JSON array: competitor brands the AUTHOR owns
+    COMPARISON_TYPE VARCHAR(50),
+    COMPETITIVE_POSITIONING VARCHAR(500),
+    BRAND_SWITCHING VARCHAR(100),
+
+    -- Customer journey (AUTHOR PERSPECTIVE ONLY - null if giving advice to others)
+    INTENTS VARCHAR(16777216),  -- Multi-label: seeking_information, providing_information, sharing_experience, comparing, praising, criticizing, buying_selling, general_discussion
+    PURCHASE_STAGE VARCHAR(50),  -- AUTHOR's own stage only
+    DECISION_DRIVERS VARCHAR(16777216),  -- AUTHOR's own decision drivers only
+    PAIN_POINTS VARCHAR(16777216),  -- AUTHOR's negative feedback aspects (uses feedback_aspects categories)
+    DELIGHT_FACTORS VARCHAR(16777216),  -- AUTHOR's positive feedback aspects (uses feedback_aspects categories)
+
+    -- Analysis notes
+    ANALYSIS_NOTES VARCHAR(16777216),
+    SARCASM_DETECTED BOOLEAN,
+
+    -- Validation results (NEW v4.0)
+    VALIDATION_PASSED BOOLEAN,
+    VALIDATION_ERRORS VARCHAR(16777216),  -- JSON array of error messages
+    VALIDATION_WARNINGS VARCHAR(16777216),  -- JSON array of warning messages
+    VALIDATION_FLAGS VARCHAR(16777216),  -- JSON array of anomaly flags
+
+    -- Platform identifier
+    PLATFORM VARCHAR(50) DEFAULT 'musora_forums',
+
+    -- Processing metadata
+    PROCESSING_SUCCESS BOOLEAN,
+    PROCESSING_STATUS VARCHAR(50),  -- NEW v4.0: completed, completed_with_flags, validation_failed, workflow_error
+    PROCESSING_ERRORS VARCHAR(16777216),
+    PROCESSED_AT TIMESTAMP_LTZ(9) DEFAULT CURRENT_TIMESTAMP(),
+    WORKFLOW_VERSION VARCHAR(20) DEFAULT '4.0'
+)
+COMMENT = 'Brand sentiment analysis results for Sabian forum posts. Schema v4.0: Added thread_context_summary, validation fields, and processing status.';

processing_brand_sentiment/main.py

@@ -0,0 +1,1088 @@
+"""
+Main execution script for brand sentiment analysis workflow.
+Orchestrates data fetching, processing, and storage using an agentic workflow.
+Supports parallel processing with multiprocessing for improved performance.
+Supports multiple data sources: forums, social media comments, or both.
+"""
+
+import json
+import os
+import logging
+import argparse
+from datetime import datetime
+import pandas as pd
+from dotenv import load_dotenv
+from multiprocessing import Pool, cpu_count
+import traceback
+from typing import Dict, Any, List
+
+from database.snowflake_connection import SnowFlakeConn
+from workflow.orchestrator import BrandAnalysisWorkflow
+from workflow.comment_orchestrator import CommentAnalysisWorkflow
+
+# Get the directory where this script is located
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Load environment variables
+load_dotenv(os.path.join(SCRIPT_DIR, '.env'))
+
+# Ensure logs directory exists
+LOGS_DIR = os.path.join(SCRIPT_DIR, 'logs')
+os.makedirs(LOGS_DIR, exist_ok=True)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler(
+            os.path.join(LOGS_DIR, f'brand_sentiment_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log')
+        ),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+
+# ============================================================
+# Configuration Loading
+# ============================================================
+
+def load_configs(config_dir: str = None) -> Dict[str, Dict]:
+    """
+    Load all configuration files.
+
+    Args:
+        config_dir: Directory containing config files
+
+    Returns:
+        Dictionary with all configurations
+    """
+    if config_dir is None:
+        config_dir = os.path.join(SCRIPT_DIR, 'config_files')
+
+    configs = {}
+
+    # Load workflow config
+    with open(os.path.join(config_dir, 'workflow_config.json'), 'r') as f:
+        configs['workflow'] = json.load(f)
+
+    # Load brand config
+    with open(os.path.join(config_dir, 'brand_config.json'), 'r') as f:
+        configs['brand'] = json.load(f)
+
+    # Load analysis categories
+    with open(os.path.join(config_dir, 'analysis_categories.json'), 'r') as f:
+        configs['categories'] = json.load(f)
+
+    return configs
+
+
+# ============================================================
+# Batch Processing Utilities
+# ============================================================
+
+def calculate_optimal_batch_size(
+    total_posts: int,
+    num_workers: int,
+    min_batch: int = 20,
+    max_batch: int = 500
+) -> int:
+    """
+    Calculate optimal batch size based on total posts and workers.
+
+    Args:
+        total_posts: Total number of posts to process
+        num_workers: Number of parallel workers
+        min_batch: Minimum batch size
+        max_batch: Maximum batch size
+
+    Returns:
+        Optimal batch size
+    """
+    if total_posts <= min_batch:
+        return total_posts
+
+    batch_size = total_posts // num_workers
+    batch_size = max(min_batch, min(max_batch, batch_size))
+
+    return batch_size
+
+
+def safe_to_json(value: Any) -> Any:
+    """
+    Safely convert a value to JSON string.
+    Handles None, NaN, lists, and already-string values.
+
+    Args:
+        value: Value to convert
+
+    Returns:
+        JSON string if list, None if null, original value otherwise
+    """
+    # Handle None and NaN
+    if value is None or (isinstance(value, float) and pd.isna(value)):
+        return None
+    # Handle lists - convert to JSON
+    if isinstance(value, list):
+        return json.dumps(value) if value else None
+    # Handle already-string values
+    if isinstance(value, str):
+        return value if value else None
+    # Return as-is for other types
+    return value
+
+
+def safe_json_list_length(value: Any) -> int:
+    """
+    Safely get the length of a JSON array string.
+    Handles None, NaN, empty strings, and invalid JSON.
+
+    Args:
+        value: Value to parse (expected JSON string of array)
+
+    Returns:
+        Length of the array, or 0 if invalid/empty
+    """
+    # Handle None and NaN
+    if value is None or (isinstance(value, float) and pd.isna(value)):
+        return 0
+    # Handle non-string values
+    if not isinstance(value, str):
+        return 0
+    # Handle empty strings
+    if not value or value == '[]' or value == 'null':
+        return 0
+    # Try to parse JSON
+    try:
+        parsed = json.loads(value)
+        return len(parsed) if isinstance(parsed, list) else 0
+    except (json.JSONDecodeError, TypeError):
+        return 0
+
+
+def calculate_batch_stats(df: pd.DataFrame) -> Dict[str, int]:
+    """
+    Calculate statistics from batch results.
+    Handles null values safely for all fields.
+
+    Args:
+        df: DataFrame with processed results
+
+    Returns:
+        Dictionary with statistics
+    """
+    stats = {
+        'relevant_count': 0,
+        'not_relevant_count': 0,
+        'products_mentioned_count': 0,
+        'competitors_mentioned_count': 0,
+        'positive_sentiment_count': 0,
+        'negative_sentiment_count': 0,
+        # Author role stats
+        'current_owner_count': 0,
+        'potential_buyer_count': 0,
+        'primary_focus_count': 0
+    }
+
+    # Handle empty dataframe
+    if df.empty:
+        return stats
+
+    # Count relevant/not relevant posts
+    if 'IS_RELEVANT' in df.columns:
+        relevant_col = df['IS_RELEVANT']
+        non_null_mask = relevant_col.notna()
+        if non_null_mask.any():
+            stats['relevant_count'] = int(relevant_col[non_null_mask].astype(bool).sum())
+            stats['not_relevant_count'] = int((~relevant_col[non_null_mask].astype(bool)).sum())
+
+    # Count product mentions using safe helper
+    if 'PRODUCTS_MENTIONED' in df.columns:
+        stats['products_mentioned_count'] = int(
+            df['PRODUCTS_MENTIONED'].apply(safe_json_list_length).sum()
+        )
+
+    # Count competitor mentions using safe helper
+    if 'COMPETITORS_MENTIONED' in df.columns:
+        stats['competitors_mentioned_count'] = int(
+            df['COMPETITORS_MENTIONED'].apply(safe_json_list_length).sum()
+        )
+
+    # Count sentiment distribution
+    if 'SENTIMENT_LEVEL' in df.columns:
+        sentiment_values = df['SENTIMENT_LEVEL'].dropna()
+        if not sentiment_values.empty:
+            stats['positive_sentiment_count'] = int(
+                sentiment_values.isin(['positive', 'very_positive']).sum()
+            )
+            stats['negative_sentiment_count'] = int(
+                sentiment_values.isin(['negative', 'very_negative']).sum()
+            )
+
+    # Count author roles
+    if 'AUTHOR_ROLE' in df.columns:
+        author_roles = df['AUTHOR_ROLE'].dropna()
+        if not author_roles.empty:
+            stats['current_owner_count'] = int((author_roles == 'current_owner').sum())
+            stats['potential_buyer_count'] = int((author_roles == 'potential_buyer').sum())
+
+    # Count mention context
+    if 'SABIAN_MENTION_CONTEXT' in df.columns:
+        mention_context = df['SABIAN_MENTION_CONTEXT'].dropna()
+        if not mention_context.empty:
+            stats['primary_focus_count'] = int((mention_context == 'primary_focus').sum())
+
+    return stats
+
+
+def aggregate_results(results: List[Dict[str, Any]]) -> Dict[str, Any]:
+    """
+    Aggregate results from multiple batches.
+
+    Args:
+        results: List of batch result dictionaries
+
+    Returns:
+        Aggregated statistics dictionary
+    """
+    aggregated = {
+        'total_processed': sum(r.get('total_processed', 0) for r in results),
+        'total_stored': sum(r.get('total_stored', 0) for r in results),
+        'failed_count': sum(r.get('failed_count', 0) for r in results),
+        'relevant_count': sum(r.get('relevant_count', 0) for r in results),
+        'not_relevant_count': sum(r.get('not_relevant_count', 0) for r in results),
+        'products_mentioned_count': sum(r.get('products_mentioned_count', 0) for r in results),
+        'competitors_mentioned_count': sum(r.get('competitors_mentioned_count', 0) for r in results),
+        'positive_sentiment_count': sum(r.get('positive_sentiment_count', 0) for r in results),
+        'negative_sentiment_count': sum(r.get('negative_sentiment_count', 0) for r in results),
+        'current_owner_count': sum(r.get('current_owner_count', 0) for r in results),
+        'potential_buyer_count': sum(r.get('potential_buyer_count', 0) for r in results),
+        'primary_focus_count': sum(r.get('primary_focus_count', 0) for r in results),
+        'failed_batches': sum(1 for r in results if not r.get('success', False))
+    }
+
+    # Log failed batches
+    failed_batches = [r for r in results if not r.get('success', False)]
+    if failed_batches:
+        logger.error(f"{len(failed_batches)} batch(es) failed:")
+        for fb in failed_batches:
+            logger.error(f"  Batch {fb.get('batch_num')}: {fb.get('error')}")
+
+    return aggregated
+
+
+# ============================================================
+# Forum Processing (existing functionality)
+# ============================================================
+
+# Columns that should be converted from lists to JSON strings
+FORUM_JSON_ARRAY_COLUMNS = [
+    'products_mentioned', 'product_attributes', 'competitors_mentioned',
+    'competitor_products_owned', 'intents', 'decision_drivers',
+    'pain_points', 'delight_factors', 'processing_errors', 'relevance_keywords_found',
+    'validation_errors', 'validation_warnings', 'validation_flags'
+]
+
+# Column mapping from forum workflow state to output table
+FORUM_COLUMN_MAPPING = {
+    'post_id': 'POST_ID',
+    'thread_id': 'THREAD_ID',
+    'post_author_id': 'POST_AUTHOR_ID',
+    'original_content': 'ORIGINAL_CONTENT',
+    'cleaned_content': 'CLEANED_CONTENT',
+    'quoted_content': 'QUOTED_CONTENT',
+    'raw_thread_context': 'THREAD_CONTEXT',
+    'thread_context_summary': 'THREAD_CONTEXT_SUMMARY',
+    'thread_title': 'THREAD_TITLE',
+    'thread_first_post': 'THREAD_FIRST_POST',
+    'post_created_at': 'POST_CREATED_AT',
+    'thread_started_at': 'THREAD_STARTED_AT',
+    'category_title': 'CATEGORY_TITLE',
+    'category_topic': 'CATEGORY_TOPIC',
+    'detected_language': 'DETECTED_LANGUAGE',
+    'language_code': 'LANGUAGE_CODE',
+    'is_english': 'IS_ENGLISH',
+    'is_relevant': 'IS_RELEVANT',
+    'relevance_confidence': 'RELEVANCE_CONFIDENCE',
+    'relevance_reason': 'RELEVANCE_REASON',
+    'author_role': 'AUTHOR_ROLE',
+    'sabian_mention_context': 'SABIAN_MENTION_CONTEXT',
+    'sentiment_level': 'SENTIMENT_LEVEL',
+    'emotion_type': 'EMOTION_TYPE',
+    'sentiment_target': 'SENTIMENT_TARGET',
+    'sentiment_confidence': 'SENTIMENT_CONFIDENCE',
+    'products_mentioned': 'PRODUCTS_MENTIONED',
+    'product_attributes': 'PRODUCT_ATTRIBUTES',
+    'competitors_mentioned': 'COMPETITORS_MENTIONED',
+    'competitor_products_owned': 'COMPETITOR_PRODUCTS_OWNED',
+    'comparison_type': 'COMPARISON_TYPE',
+    'competitive_positioning': 'COMPETITIVE_POSITIONING',
+    'brand_switching': 'BRAND_SWITCHING',
+    'intents': 'INTENTS',
+    'purchase_stage': 'PURCHASE_STAGE',
+    'decision_drivers': 'DECISION_DRIVERS',
+    'pain_points': 'PAIN_POINTS',
+    'delight_factors': 'DELIGHT_FACTORS',
+    'analysis_notes': 'ANALYSIS_NOTES',
+    'sarcasm_detected': 'SARCASM_DETECTED',
+    'validation_passed': 'VALIDATION_PASSED',
+    'validation_errors': 'VALIDATION_ERRORS',
+    'validation_warnings': 'VALIDATION_WARNINGS',
+    'validation_flags': 'VALIDATION_FLAGS',
+    'success': 'PROCESSING_SUCCESS',
+    'processing_status': 'PROCESSING_STATUS',
+    'processing_errors': 'PROCESSING_ERRORS'
+}
+
+
+def prepare_forum_output_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Prepare forum output DataFrame with proper column mapping.
+
+    Args:
+        df: DataFrame with processing results
+
+    Returns:
+        DataFrame ready for Snowflake storage
+    """
+    output_df = pd.DataFrame()
+
+    for source_col, target_col in FORUM_COLUMN_MAPPING.items():
+        if source_col in df.columns:
+            value = df[source_col].copy()
+            if source_col in FORUM_JSON_ARRAY_COLUMNS:
+                value = value.apply(safe_to_json)
+            output_df[target_col] = value
+        else:
+            output_df[target_col] = None
+
+    # Add metadata
+    output_df['PLATFORM'] = 'musora_forums'
+    output_df['PROCESSED_AT'] = datetime.now()
+    output_df['WORKFLOW_VERSION'] = '4.0'
+
+    return output_df
+
+
+def process_forum_batch_worker(batch_data: tuple) -> Dict[str, Any]:
+    """
+    Worker function to process a single batch of forum posts.
+    Runs in a separate process.
+
+    Args:
+        batch_data: Tuple containing (batch_num, posts, configs, api_key, overwrite_first_batch, output_config)
+
+    Returns:
+        Dictionary with batch statistics
+    """
+    batch_num, posts, configs, api_key, overwrite_first_batch, output_config = batch_data
+
+    worker_logger = logging.getLogger(f"ForumWorker-{batch_num}")
+
+    try:
+        worker_logger.info(f"Forum Batch {batch_num}: Starting processing of {len(posts)} posts")
+
+        # Initialize Snowflake connection for this worker
+        snowflake = SnowFlakeConn()
+
+        # Initialize workflow for this worker
+        workflow = BrandAnalysisWorkflow(
+            workflow_config=configs['workflow'],
+            brand_config=configs['brand'],
+            analysis_categories=configs['categories'],
+            api_key=api_key
+        )
+
+        # Process posts
+        results = workflow.process_batch(posts)
+
+        # Convert to DataFrame
+        results_df = pd.DataFrame(results)
+
+        # Filter successful results
+        initial_count = len(results_df)
+        df_successful = results_df[results_df['success'] == True].copy()
+        failed_count = initial_count - len(df_successful)
+
+        worker_logger.info(f"Forum Batch {batch_num}: Processed {initial_count} posts, {len(df_successful)} successful")
+
+        # Prepare output DataFrame
+        output_df = prepare_forum_output_dataframe(df_successful)
+
+        # Store results
+        if len(output_df) > 0:
+            overwrite = overwrite_first_batch and batch_num == 1
+
+            snowflake.store_df_to_snowflake(
+                table_name=output_config['table_name'],
+                dataframe=output_df,
+                database=output_config['database'],
+                schema=output_config['schema'],
+                overwrite=overwrite
+            )
+
+            worker_logger.info(f"Forum Batch {batch_num}: Stored {len(output_df)} records to Snowflake")
+        else:
+            worker_logger.warning(f"Forum Batch {batch_num}: No successful records to store")
+
+        # Close connection
+        snowflake.close_connection()
+
+        # Calculate statistics
+        stats = calculate_batch_stats(output_df)
+        stats.update({
+            'batch_num': batch_num,
+            'success': True,
+            'total_processed': initial_count,
+            'total_stored': len(output_df),
+            'failed_count': failed_count,
+            'error': None
+        })
+
+        return stats
+
+    except Exception as e:
+        error_msg = f"Forum Batch {batch_num} failed: {str(e)}"
+        worker_logger.error(error_msg)
+        worker_logger.error(traceback.format_exc())
+
+        return {
+            'batch_num': batch_num,
+            'success': False,
+            'total_processed': len(posts),
+            'total_stored': 0,
+            'failed_count': len(posts),
+            'error': error_msg
+        }
+
+
+# ============================================================
+# Comment Processing (new functionality)
+# ============================================================
+
+# Columns that should be converted from lists to JSON strings (same analysis fields)
+COMMENT_JSON_ARRAY_COLUMNS = [
+    'products_mentioned', 'product_attributes', 'competitors_mentioned',
+    'competitor_products_owned', 'intents', 'decision_drivers',
+    'pain_points', 'delight_factors', 'processing_errors', 'relevance_keywords_found',
+    'validation_errors', 'validation_warnings', 'validation_flags'
+]
+
+# Column mapping from comment workflow state to output table
+COMMENT_COLUMN_MAPPING = {
+    # Comment-specific identifiers
+    'comment_sk': 'COMMENT_SK',
+    'comment_id': 'COMMENT_ID',
+    'original_text': 'ORIGINAL_TEXT',
+    'platform': 'PLATFORM',
+    'comment_timestamp': 'COMMENT_TIMESTAMP',
+    'author_name': 'AUTHOR_NAME',
+    'author_id': 'AUTHOR_ID',
+    'content_sk': 'CONTENT_SK',
+    'content_id': 'CONTENT_ID',
+    'content_description': 'CONTENT_DESCRIPTION',
+    'channel_sk': 'CHANNEL_SK',
+    'channel_name': 'CHANNEL_NAME',
+    'channel_display_name': 'CHANNEL_DISPLAY_NAME',
+    'parent_comment_id': 'PARENT_COMMENT_ID',
+    'parent_comment_text': 'PARENT_COMMENT_TEXT',
+    # Analysis fields (same as forums)
+    'detected_language': 'DETECTED_LANGUAGE',
+    'language_code': 'LANGUAGE_CODE',
+    'is_english': 'IS_ENGLISH',
+    'is_relevant': 'IS_RELEVANT',
+    'relevance_confidence': 'RELEVANCE_CONFIDENCE',
+    'relevance_reason': 'RELEVANCE_REASON',
+    'author_role': 'AUTHOR_ROLE',
+    'sabian_mention_context': 'SABIAN_MENTION_CONTEXT',
+    'sentiment_level': 'SENTIMENT_LEVEL',
+    'emotion_type': 'EMOTION_TYPE',
+    'sentiment_target': 'SENTIMENT_TARGET',
+    'sentiment_confidence': 'SENTIMENT_CONFIDENCE',
+    'products_mentioned': 'PRODUCTS_MENTIONED',
+    'product_attributes': 'PRODUCT_ATTRIBUTES',
+    'purchase_stage': 'PURCHASE_STAGE',
+    'competitors_mentioned': 'COMPETITORS_MENTIONED',
+    'competitor_products_owned': 'COMPETITOR_PRODUCTS_OWNED',
+    'comparison_type': 'COMPARISON_TYPE',
+    'competitive_positioning': 'COMPETITIVE_POSITIONING',
+    'brand_switching': 'BRAND_SWITCHING',
+    'intents': 'INTENTS',
+    'decision_drivers': 'DECISION_DRIVERS',
+    'pain_points': 'PAIN_POINTS',
+    'delight_factors': 'DELIGHT_FACTORS',
+    'analysis_notes': 'ANALYSIS_NOTES',
+    'sarcasm_detected': 'SARCASM_DETECTED',
+    'validation_passed': 'VALIDATION_PASSED',
+    'validation_errors': 'VALIDATION_ERRORS',
+    'validation_warnings': 'VALIDATION_WARNINGS',
+    'validation_flags': 'VALIDATION_FLAGS',
+    'success': 'PROCESSING_SUCCESS',
+    'processing_status': 'PROCESSING_STATUS',
+    'processing_errors': 'PROCESSING_ERRORS'
+}
+
+
+def prepare_comment_output_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Prepare comment output DataFrame with proper column mapping.
+
+    Args:
+        df: DataFrame with processing results
+
+    Returns:
+        DataFrame ready for Snowflake storage
+    """
+    output_df = pd.DataFrame()
+
+    for source_col, target_col in COMMENT_COLUMN_MAPPING.items():
+        if source_col in df.columns:
+            value = df[source_col].copy()
+            if source_col in COMMENT_JSON_ARRAY_COLUMNS:
+                value = value.apply(safe_to_json)
+            output_df[target_col] = value
+        else:
+            output_df[target_col] = None
+
+    # Add metadata
+    output_df['PROCESSED_AT'] = datetime.now()
+    output_df['WORKFLOW_VERSION'] = '4.0'
+
+    return output_df
+
+
+def process_comment_batch_worker(batch_data: tuple) -> Dict[str, Any]:
+    """
+    Worker function to process a single batch of social media comments.
+    Runs in a separate process.
+
+    Args:
+        batch_data: Tuple containing (batch_num, comments, configs, api_key, overwrite_first_batch, output_config)
+
+    Returns:
+        Dictionary with batch statistics
+    """
+    batch_num, comments, configs, api_key, overwrite_first_batch, output_config = batch_data
+
+    worker_logger = logging.getLogger(f"CommentWorker-{batch_num}")
+
+    try:
+        worker_logger.info(f"Comment Batch {batch_num}: Starting processing of {len(comments)} comments")
+
+        # Initialize Snowflake connection for this worker
+        snowflake = SnowFlakeConn()
+
+        # Initialize comment workflow for this worker
+        workflow = CommentAnalysisWorkflow(
+            workflow_config=configs['workflow'],
+            brand_config=configs['brand'],
+            analysis_categories=configs['categories'],
+            api_key=api_key
+        )
+
+        # Process comments
+        results = workflow.process_batch(comments)
+
+        # Convert to DataFrame
+        results_df = pd.DataFrame(results)
+
+        # Filter successful results
+        initial_count = len(results_df)
+        df_successful = results_df[results_df['success'] == True].copy()
+        failed_count = initial_count - len(df_successful)
+
+        worker_logger.info(f"Comment Batch {batch_num}: Processed {initial_count} comments, {len(df_successful)} successful")
+
+        # Prepare output DataFrame
+        output_df = prepare_comment_output_dataframe(df_successful)
+
+        # Store results
+        if len(output_df) > 0:
+            overwrite = overwrite_first_batch and batch_num == 1
+
+            snowflake.store_df_to_snowflake(
+                table_name=output_config['table_name'],
+                dataframe=output_df,
+                database=output_config['database'],
+                schema=output_config['schema'],
+                overwrite=overwrite
+            )
+
+            worker_logger.info(f"Comment Batch {batch_num}: Stored {len(output_df)} records to Snowflake")
+        else:
+            worker_logger.warning(f"Comment Batch {batch_num}: No successful records to store")
+
+        # Close connection
+        snowflake.close_connection()
+
+        # Calculate statistics
+        stats = calculate_batch_stats(output_df)
+        stats.update({
+            'batch_num': batch_num,
+            'success': True,
+            'total_processed': initial_count,
+            'total_stored': len(output_df),
+            'failed_count': failed_count,
+            'error': None
+        })
+
+        return stats
+
+    except Exception as e:
+        error_msg = f"Comment Batch {batch_num} failed: {str(e)}"
+        worker_logger.error(error_msg)
+        worker_logger.error(traceback.format_exc())
+
+        return {
+            'batch_num': batch_num,
+            'success': False,
+            'total_processed': len(comments),
+            'total_stored': 0,
+            'failed_count': len(comments),
+            'error': error_msg
+        }
+
+
+# ============================================================
+# Main Processor Class
+# ============================================================
+
+class BrandSentimentProcessor:
+    """
+    Main processor class that orchestrates the entire workflow.
+    Supports processing forums, social media comments, or both.
+    """
+
+    def __init__(self, config_dir: str = None):
+        """
+        Initialize the processor.
+
+        Args:
+            config_dir: Directory containing configuration files
+        """
+        # Load configurations
+        self.configs = load_configs(config_dir)
+
+        # Initialize Snowflake connection
+        self.snowflake = SnowFlakeConn()
+
+        # Get OpenAI API key
+        self.api_key = os.getenv("OPENAI_API_KEY")
+        if not self.api_key:
+            raise ValueError("OPENAI_API_KEY not found in environment variables")
+
+        # Get output configurations
+        self.forum_output_config = self.configs['workflow'].get('output', {
+            'table_name': 'SABIAN_BRAND_ANALYSIS',
+            'database': 'SOCIAL_MEDIA_DB',
+            'schema': 'ML_FEATURES'
+        })
+
+        self.comment_output_config = self.configs['workflow'].get('comments_output', {
+            'table_name': 'SABIAN_BRAND_ANALYSIS_COMMENTS',
+            'database': 'SOCIAL_MEDIA_DB',
+            'schema': 'ML_FEATURES'
+        })
+
+        logger.info("BrandSentimentProcessor initialized successfully")
+
+    def fetch_forum_posts(self, limit: int = None) -> pd.DataFrame:
+        """
+        Fetch forum posts from Snowflake.
+
+        Args:
+            limit: Optional limit on number of posts
+
+        Returns:
+            DataFrame containing post data
+        """
+        logger.info("Fetching forum posts...")
+
+        sql_path = os.path.join(SCRIPT_DIR, 'database', 'sql', 'fetch_forum_posts.sql')
+        df = self.snowflake.fetch_forum_posts_with_context(sql_path, limit)
+
+        logger.info(f"Fetched {len(df)} forum posts")
+        return df
+
+    def fetch_comments(self, limit: int = None) -> pd.DataFrame:
+        """
+        Fetch social media comments from Snowflake.
+
+        Args:
+            limit: Optional limit on number of comments
+
+        Returns:
+            DataFrame containing comment data
+        """
+        logger.info("Fetching social media comments...")
+
+        sql_path = os.path.join(SCRIPT_DIR, 'database', 'sql', 'fetch_comments.sql')
+        df = self.snowflake.fetch_comments(sql_path, limit)
+
+        logger.info(f"Fetched {len(df)} social media comments")
+        return df
+
+    def calculate_num_workers(self) -> int:
+        """
+        Calculate number of parallel workers.
+
+        Returns:
+            Number of workers
+        """
+        parallel_config = self.configs['workflow'].get('workflow', {}).get('parallel_processing', {})
+        max_workers = parallel_config.get('max_workers', 5)
+
+        num_cpus = cpu_count()
+        num_workers = max(1, min(max_workers, num_cpus - 2))
+
+        logger.info(f"Using {num_workers} parallel workers (CPU count: {num_cpus})")
+        return num_workers
+
+    # ---- Forum Processing ----
+
+    def process_forums_parallel(self, df: pd.DataFrame, overwrite: bool = False) -> Dict[str, Any]:
+        """
+        Process forum posts using parallel workers.
+
+        Args:
+            df: DataFrame containing posts
+            overwrite: Whether to overwrite existing table
+
+        Returns:
+            Dictionary with aggregated statistics
+        """
+        posts = df.to_dict('records')
+        total_posts = len(posts)
+
+        logger.info(f"Processing {total_posts} forum posts using parallel processing...")
+
+        num_workers = self.calculate_num_workers()
+
+        parallel_config = self.configs['workflow'].get('workflow', {}).get('parallel_processing', {})
+        min_batch = parallel_config.get('min_batch_size', 20)
+        max_batch = parallel_config.get('max_batch_size', 400)
+
+        batch_size = calculate_optimal_batch_size(total_posts, num_workers, min_batch, max_batch)
+        logger.info(f"Forum batch size: {batch_size}")
+
+        # Create batches
+        batches = []
+        for i in range(0, total_posts, batch_size):
+            batch = posts[i:i + batch_size]
+            batch_num = (i // batch_size) + 1
+            batches.append((batch_num, batch, self.configs, self.api_key, overwrite, self.forum_output_config))
+
+        total_batches = len(batches)
+        logger.info(f"Split into {total_batches} forum batches")
+
+        # Process in parallel
+        with Pool(processes=num_workers) as pool:
+            results = pool.map(process_forum_batch_worker, batches)
+
+        return aggregate_results(results)
+
+    def process_forums_sequential(self, df: pd.DataFrame, overwrite: bool = False) -> Dict[str, Any]:
+        """
+        Process forum posts sequentially (for debugging).
+
+        Args:
+            df: DataFrame containing posts
+            overwrite: Whether to overwrite existing table
+
+        Returns:
+            Dictionary with statistics
+        """
+        logger.info(f"Processing {len(df)} forum posts using sequential processing...")
+
+        posts = df.to_dict('records')
+        batch_data = (1, posts, self.configs, self.api_key, overwrite, self.forum_output_config)
+        result = process_forum_batch_worker(batch_data)
+
+        return {
+            'total_processed': result.get('total_processed', 0),
+            'total_stored': result.get('total_stored', 0),
+            'failed_count': result.get('failed_count', 0),
+            'relevant_count': result.get('relevant_count', 0),
+            'not_relevant_count': result.get('not_relevant_count', 0),
+            'products_mentioned_count': result.get('products_mentioned_count', 0),
+            'competitors_mentioned_count': result.get('competitors_mentioned_count', 0),
+            'positive_sentiment_count': result.get('positive_sentiment_count', 0),
+            'negative_sentiment_count': result.get('negative_sentiment_count', 0),
+            'current_owner_count': result.get('current_owner_count', 0),
+            'potential_buyer_count': result.get('potential_buyer_count', 0),
+            'primary_focus_count': result.get('primary_focus_count', 0),
+            'failed_batches': 0 if result.get('success', False) else 1
+        }
+
+    # ---- Comment Processing ----
+
+    def process_comments_parallel(self, df: pd.DataFrame, overwrite: bool = False) -> Dict[str, Any]:
+        """
+        Process social media comments using parallel workers.
+
+        Args:
+            df: DataFrame containing comments
+            overwrite: Whether to overwrite existing table
+
+        Returns:
+            Dictionary with aggregated statistics
+        """
+        comments = df.to_dict('records')
+        total_comments = len(comments)
+
+        logger.info(f"Processing {total_comments} comments using parallel processing...")
+
+        num_workers = self.calculate_num_workers()
+
+        parallel_config = self.configs['workflow'].get('workflow', {}).get('parallel_processing', {})
+        min_batch = parallel_config.get('min_batch_size', 20)
+        max_batch = parallel_config.get('max_batch_size', 400)
+
+        batch_size = calculate_optimal_batch_size(total_comments, num_workers, min_batch, max_batch)
+        logger.info(f"Comment batch size: {batch_size}")
+
+        # Create batches
+        batches = []
+        for i in range(0, total_comments, batch_size):
+            batch = comments[i:i + batch_size]
+            batch_num = (i // batch_size) + 1
+            batches.append((batch_num, batch, self.configs, self.api_key, overwrite, self.comment_output_config))
+
+        total_batches = len(batches)
+        logger.info(f"Split into {total_batches} comment batches")
+
+        # Process in parallel
+        with Pool(processes=num_workers) as pool:
+            results = pool.map(process_comment_batch_worker, batches)
+
+        return aggregate_results(results)
+
+    def process_comments_sequential(self, df: pd.DataFrame, overwrite: bool = False) -> Dict[str, Any]:
+        """
+        Process social media comments sequentially (for debugging).
+
+        Args:
+            df: DataFrame containing comments
+            overwrite: Whether to overwrite existing table
+
+        Returns:
+            Dictionary with statistics
+        """
+        logger.info(f"Processing {len(df)} comments using sequential processing...")
+
+        comments = df.to_dict('records')
+        batch_data = (1, comments, self.configs, self.api_key, overwrite, self.comment_output_config)
+        result = process_comment_batch_worker(batch_data)
+
+        return {
+            'total_processed': result.get('total_processed', 0),
+            'total_stored': result.get('total_stored', 0),
+            'failed_count': result.get('failed_count', 0),
+            'relevant_count': result.get('relevant_count', 0),
+            'not_relevant_count': result.get('not_relevant_count', 0),
+            'products_mentioned_count': result.get('products_mentioned_count', 0),
+            'competitors_mentioned_count': result.get('competitors_mentioned_count', 0),
+            'positive_sentiment_count': result.get('positive_sentiment_count', 0),
+            'negative_sentiment_count': result.get('negative_sentiment_count', 0),
+            'current_owner_count': result.get('current_owner_count', 0),
+            'potential_buyer_count': result.get('potential_buyer_count', 0),
+            'primary_focus_count': result.get('primary_focus_count', 0),
+            'failed_batches': 0 if result.get('success', False) else 1
+        }
+
+    # ---- Unified Processing ----
+
+    def _log_source_summary(self, source_name: str, stats: Dict[str, Any], processing_time: float) -> None:
+        """
+        Log processing summary for a data source.
+
+        Args:
+            source_name: Name of the data source
+            stats: Processing statistics
+            processing_time: Time taken in seconds
+        """
+        logger.info(f"  --- {source_name} ---")
+        logger.info(f"  Total processed: {stats.get('total_processed', 0)}")
+        logger.info(f"  Successfully stored: {stats.get('total_stored', 0)}")
+        logger.info(f"  Failed: {stats.get('failed_count', 0)}")
+        logger.info(f"  Relevant: {stats.get('relevant_count', 0)}")
+        logger.info(f"  Not relevant: {stats.get('not_relevant_count', 0)}")
+        logger.info(f"  Product mentions: {stats.get('products_mentioned_count', 0)}")
+        logger.info(f"  Competitor mentions: {stats.get('competitors_mentioned_count', 0)}")
+        logger.info(f"  Positive sentiment: {stats.get('positive_sentiment_count', 0)}")
+        logger.info(f"  Negative sentiment: {stats.get('negative_sentiment_count', 0)}")
+        logger.info(f"  Current owners: {stats.get('current_owner_count', 0)}")
+        logger.info(f"  Potential buyers: {stats.get('potential_buyer_count', 0)}")
+        logger.info(f"  Primary focus: {stats.get('primary_focus_count', 0)}")
+        if stats.get('failed_batches', 0) > 0:
+            logger.info(f"  Failed batches: {stats['failed_batches']}")
+        logger.info(f"  Processing time: {processing_time:.2f} seconds")
+        if stats.get('total_processed', 0) > 0:
+            logger.info(f"  Average per item: {processing_time / stats['total_processed']:.2f} seconds")
+
+    def run(
+        self,
+        limit: int = None,
+        overwrite: bool = False,
+        sequential: bool = False,
+        data_source: str = 'all'
+    ):
+        """
+        Run the complete processing pipeline.
+
+        Args:
+            limit: Optional limit on items to process per source
+            overwrite: Whether to overwrite existing table
+            sequential: Use sequential processing instead of parallel
+            data_source: Which data source to process ('forums', 'comments', 'all')
+        """
+        try:
+            logger.info("=" * 80)
+            logger.info("Starting Brand Sentiment Analysis Workflow")
+            logger.info(f"Brand: {self.configs['brand'].get('brand', {}).get('name', 'Unknown')}")
+            logger.info(f"Mode: {'SEQUENTIAL' if sequential else 'PARALLEL'}")
+            logger.info(f"Data source: {data_source}")
+            logger.info("=" * 80)
+
+            process_forums = data_source in ('forums', 'all')
+            process_comments = data_source in ('comments', 'all')
+
+            # Track results for summary
+            forum_stats = None
+            forum_time = 0.0
+            comment_stats = None
+            comment_time = 0.0
+
+            # ---- Process Forums ----
+            if process_forums:
+                logger.info("-" * 40)
+                logger.info("Processing FORUMS")
+                logger.info("-" * 40)
+
+                df_posts = self.fetch_forum_posts(limit)
+
+                if df_posts.empty:
+                    logger.warning("No forum posts to process")
+                else:
+                    start_time = datetime.now()
+
+                    if sequential:
+                        forum_stats = self.process_forums_sequential(df_posts, overwrite)
+                    else:
+                        forum_stats = self.process_forums_parallel(df_posts, overwrite)
+
+                    forum_time = (datetime.now() - start_time).total_seconds()
+
+            # ---- Process Comments ----
+            if process_comments:
+                logger.info("-" * 40)
+                logger.info("Processing SOCIAL MEDIA COMMENTS")
+                logger.info("-" * 40)
+
+                df_comments = self.fetch_comments(limit)
+
+                if df_comments.empty:
+                    logger.warning("No social media comments to process")
+                else:
+                    start_time = datetime.now()
+
+                    if sequential:
+                        comment_stats = self.process_comments_sequential(df_comments, overwrite)
+                    else:
+                        comment_stats = self.process_comments_parallel(df_comments, overwrite)
+
+                    comment_time = (datetime.now() - start_time).total_seconds()
+
+            # ---- Summary ----
+            logger.info("=" * 80)
+            logger.info("Processing Summary:")
+            logger.info(f"  Mode: {'Sequential' if sequential else 'Parallel'}")
+            logger.info(f"  Data source: {data_source}")
+
+            if forum_stats is not None:
+                self._log_source_summary("Forums", forum_stats, forum_time)
+
+            if comment_stats is not None:
+                self._log_source_summary("Social Media Comments", comment_stats, comment_time)
+
+            logger.info("=" * 80)
+
+        except Exception as e:
+            logger.error(f"Error in workflow execution: {str(e)}", exc_info=True)
+            raise
+
+        finally:
+            self.snowflake.close_connection()
+            logger.info("Snowflake connection closed")
+
+
+# ============================================================
+# Legacy compatibility - keep old function names working
+# ============================================================
+
+def prepare_output_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """Legacy wrapper for forum output preparation."""
+    return prepare_forum_output_dataframe(df)
+
+
+def process_batch_worker(batch_data: tuple) -> Dict[str, Any]:
+    """Legacy wrapper for forum batch worker."""
+    return process_forum_batch_worker(batch_data)
+
+
+# ============================================================
+# Main Entry Point
+# ============================================================
+
+def main():
+    """Main entry point."""
+    parser = argparse.ArgumentParser(
+        description="Brand Sentiment Analysis - Analyze forum posts and social media comments for brand intelligence"
+    )
+    parser.add_argument(
+        '--limit',
+        type=int,
+        default=None,
+        help='Limit number of items to process per source (default: all unprocessed)'
+    )
+    parser.add_argument(
+        '--overwrite',
+        action='store_true',
+        default=False,
+        help='Overwrite existing Snowflake table (default: append)'
+    )
+    parser.add_argument(
+        '--sequential',
+        action='store_true',
+        default=False,
+        help='Use sequential processing instead of parallel (for debugging)'
+    )
+    parser.add_argument(
+        '--config-dir',
+        type=str,
+        default=None,
+        help='Path to configuration directory (default: config_files/)'
+    )
+    parser.add_argument(
+        '--data-source',
+        type=str,
+        choices=['forums', 'comments', 'all'],
+        default='all',
+        help='Data source to process: forums, comments, or all (default: all)'
+    )
+
+    args = parser.parse_args()
+
+    # Initialize and run
+    processor = BrandSentimentProcessor(config_dir=args.config_dir)
+    processor.run(
+        limit=args.limit,
+        overwrite=args.overwrite,
+        sequential=args.sequential,
+        data_source=args.data_source
+    )
+
+
+if __name__ == "__main__":
+    main()

processing_brand_sentiment/utils/__init__.py

@@ -0,0 +1,8 @@
+"""
+Utilities module for brand sentiment analysis.
+Contains HTML parsing and other helper functions.
+"""
+
+from .html_parser import HTMLParser
+
+__all__ = ['HTMLParser']

processing_brand_sentiment/utils/html_parser.py

@@ -0,0 +1,253 @@
+"""
+HTML Parser utility for extracting content from forum posts.
+Handles the complex HTML structure where replies contain quoted parent content.
+"""
+
+import re
+import html
+from typing import Dict, Optional, Tuple
+from bs4 import BeautifulSoup
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class HTMLParser:
+    """
+    Parses HTML content from forum posts to extract actual reply content
+    and quoted parent content separately.
+    """
+
+    def __init__(self):
+        """Initialize the HTML parser."""
+        pass
+
+    def parse_post_content(self, html_content: str) -> Dict[str, Optional[str]]:
+        """
+        Parse HTML post content to extract reply and quoted content.
+
+        The forum posts have a structure where:
+        - <blockquote> contains the quoted parent post
+        - Content outside blockquote is the actual reply
+
+        Example input:
+        <blockquote><span class="post-id">125015</span>
+        <p class="quote-heading"><strong>JackO</strong><em> - Feb 3, 2015</em></p>
+        <br /><p>Parent content here...</p></blockquote>
+        <br /><p>Actual reply content here...</p>
+
+        Args:
+            html_content: Raw HTML content from POST_CONTENT field
+
+        Returns:
+            Dictionary with:
+            - reply_content: The actual reply text (cleaned)
+            - quoted_content: The quoted parent text (cleaned), if any
+            - quoted_author: Author of the quoted post, if any
+            - quoted_date: Date of the quoted post, if any
+            - has_quote: Boolean indicating if post contains a quote
+        """
+        if not html_content or not html_content.strip():
+            return {
+                "reply_content": "",
+                "quoted_content": None,
+                "quoted_author": None,
+                "quoted_date": None,
+                "has_quote": False
+            }
+
+        try:
+            soup = BeautifulSoup(html_content, 'html.parser')
+
+            # Extract quoted content from blockquotes
+            quoted_content = None
+            quoted_author = None
+            quoted_date = None
+            has_quote = False
+
+            blockquotes = soup.find_all('blockquote')
+
+            if blockquotes:
+                has_quote = True
+                quote_parts = []
+
+                for blockquote in blockquotes:
+                    # Extract quote heading info (author and date)
+                    quote_heading = blockquote.find('p', class_='quote-heading')
+                    if quote_heading:
+                        author_tag = quote_heading.find('strong')
+                        if author_tag:
+                            quoted_author = author_tag.get_text(strip=True)
+
+                        date_tag = quote_heading.find('em')
+                        if date_tag:
+                            quoted_date = date_tag.get_text(strip=True).lstrip(' - ')
+
+                    # Get the quote text content (excluding heading)
+                    # Remove the heading first to get just the content
+                    if quote_heading:
+                        quote_heading.decompose()
+
+                    # Remove post-id spans
+                    for post_id_span in blockquote.find_all('span', class_='post-id'):
+                        post_id_span.decompose()
+
+                    quote_text = self._clean_text(blockquote.get_text())
+                    if quote_text:
+                        quote_parts.append(quote_text)
+
+                    # Remove the blockquote from the soup to get remaining content
+                    blockquote.decompose()
+
+                quoted_content = " ".join(quote_parts) if quote_parts else None
+
+            # Get the remaining content (actual reply)
+            reply_content = self._clean_text(soup.get_text())
+
+            return {
+                "reply_content": reply_content,
+                "quoted_content": quoted_content,
+                "quoted_author": quoted_author,
+                "quoted_date": quoted_date,
+                "has_quote": has_quote
+            }
+
+        except Exception as e:
+            logger.warning(f"Error parsing HTML content: {e}")
+            # Fallback: try to extract text directly
+            return {
+                "reply_content": self._clean_text(self._strip_html_tags(html_content)),
+                "quoted_content": None,
+                "quoted_author": None,
+                "quoted_date": None,
+                "has_quote": False
+            }
+
+    def _clean_text(self, text: str) -> str:
+        """
+        Clean extracted text by removing extra whitespace and normalizing.
+
+        Args:
+            text: Raw text to clean
+
+        Returns:
+            Cleaned text
+        """
+        if not text:
+            return ""
+
+        # Decode HTML entities
+        text = html.unescape(text)
+
+        # Replace multiple whitespace with single space
+        text = re.sub(r'\s+', ' ', text)
+
+        # Strip leading/trailing whitespace
+        text = text.strip()
+
+        return text
+
+    def _strip_html_tags(self, html_content: str) -> str:
+        """
+        Fallback method to strip HTML tags if BeautifulSoup fails.
+
+        Args:
+            html_content: HTML content
+
+        Returns:
+            Text without HTML tags
+        """
+        # Remove HTML tags
+        clean = re.sub(r'<[^>]+>', ' ', html_content)
+        # Decode entities
+        clean = html.unescape(clean)
+        # Clean whitespace
+        clean = re.sub(r'\s+', ' ', clean)
+        return clean.strip()
+
+    def extract_plain_text(self, html_content: str) -> str:
+        """
+        Extract plain text from HTML content, preserving readability.
+
+        Args:
+            html_content: HTML content
+
+        Returns:
+            Plain text version
+        """
+        if not html_content:
+            return ""
+
+        try:
+            soup = BeautifulSoup(html_content, 'html.parser')
+
+            # Add newlines for block elements
+            for br in soup.find_all('br'):
+                br.replace_with('\n')
+            for p in soup.find_all('p'):
+                p.append('\n')
+
+            text = soup.get_text()
+            return self._clean_text(text)
+
+        except Exception as e:
+            logger.warning(f"Error extracting plain text: {e}")
+            return self._clean_text(self._strip_html_tags(html_content))
+
+    def build_thread_context(
+        self,
+        thread_title: Optional[str],
+        first_post_content: Optional[str],
+        category_title: Optional[str] = None,
+        category_topic: Optional[str] = None
+    ) -> str:
+        """
+        Build a context string from thread information.
+
+        Args:
+            thread_title: Title of the discussion thread
+            first_post_content: Content of the first post in the thread
+            category_title: Category title
+            category_topic: Category topic
+
+        Returns:
+            Formatted context string
+        """
+        context_parts = []
+
+        if category_title:
+            context_parts.append(f"Category: {category_title}")
+
+        if category_topic:
+            context_parts.append(f"Topic: {category_topic}")
+
+        if thread_title:
+            context_parts.append(f"Thread: {thread_title}")
+
+        if first_post_content:
+            # Parse and clean the first post content
+            parsed = self.parse_post_content(first_post_content)
+            first_post_text = parsed.get("reply_content", "")
+            if first_post_text:
+                # Truncate if too long
+                if len(first_post_text) > 500:
+                    first_post_text = first_post_text[:500] + "..."
+                context_parts.append(f"Original discussion: {first_post_text}")
+
+        return " | ".join(context_parts) if context_parts else ""
+
+    def is_empty_content(self, html_content: str) -> bool:
+        """
+        Check if HTML content is effectively empty.
+
+        Args:
+            html_content: HTML content to check
+
+        Returns:
+            True if content is empty or contains no meaningful text
+        """
+        if not html_content:
+            return True
+
+        text = self.extract_plain_text(html_content)
+        return len(text.strip()) == 0

processing_brand_sentiment/workflow/__init__.py

@@ -0,0 +1,10 @@
+"""
+Workflow module for brand sentiment analysis.
+Contains the LangGraph orchestrators and agent implementations.
+Supports both forum posts and social media comments.
+"""
+
+from .orchestrator import BrandAnalysisWorkflow
+from .comment_orchestrator import CommentAnalysisWorkflow
+
+__all__ = ['BrandAnalysisWorkflow', 'CommentAnalysisWorkflow']

processing_brand_sentiment/workflow/agents/__init__.py

@@ -0,0 +1,39 @@
+"""
+Agents module for brand sentiment analysis v4.0.
+
+Contains specialized agents for the 4-stage pipeline:
+1. ContentPreprocessorAgent - HTML parsing, cleaning, keyword detection (forums)
+   CommentPreprocessorAgent - Plain text cleaning, keyword detection (comments)
+2. SabianRelevanceExtractionAgent - Relevance + fact extraction
+3. SabianSentimentAnalyzerAgent - Deep sentiment analysis
+4. OutputValidatorAgent - Rule-based validation
+"""
+
+from .base_agent import BaseAgent
+from .content_preprocessor_agent import ContentPreprocessorAgent
+from .comment_preprocessor_agent import CommentPreprocessorAgent
+from .sabian_relevance_extraction_agent import SabianRelevanceExtractionAgent
+from .sabian_sentiment_analyzer_agent import SabianSentimentAnalyzerAgent
+from .output_validator_agent import OutputValidatorAgent
+
+# Legacy imports for backward compatibility
+from .preprocessor_agent import PreprocessorAgent
+from .relevance_validator_agent import RelevanceValidatorAgent
+from .sabian_analyzer_agent import SabianAnalyzerAgent
+
+__all__ = [
+    # Base
+    'BaseAgent',
+
+    # New agents (v4.0)
+    'ContentPreprocessorAgent',
+    'CommentPreprocessorAgent',
+    'SabianRelevanceExtractionAgent',
+    'SabianSentimentAnalyzerAgent',
+    'OutputValidatorAgent',
+
+    # Legacy agents (for backward compatibility)
+    'PreprocessorAgent',
+    'RelevanceValidatorAgent',
+    'SabianAnalyzerAgent'
+]

processing_brand_sentiment/workflow/agents/base_agent.py

@@ -0,0 +1,169 @@
+"""
+Base Agent class for all agents in the brand sentiment analysis workflow.
+Provides a common interface and structure for extensibility.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Dict, Any, Optional
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAgent(ABC):
+    """
+    Abstract base class for all agents in the brand sentiment analysis workflow.
+    Provides common functionality and enforces consistent interface.
+    """
+
+    def __init__(self, name: str, config: Dict[str, Any]):
+        """
+        Initialize the base agent.
+
+        Args:
+            name: Name of the agent
+            config: Configuration dictionary for the agent
+        """
+        self.name = name
+        self.config = config
+        self.model = config.get("model", "gpt-5-nano")
+        self.temperature = config.get("temperature", 0.2)
+        self.max_retries = config.get("max_retries", 3)
+        logger.info(f"Initialized {self.name} with model {self.model}")
+
+    @abstractmethod
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process input data and return results.
+        This method must be implemented by all concrete agent classes.
+
+        Args:
+            input_data: Dictionary containing input data for processing
+
+        Returns:
+            Dictionary containing processing results
+        """
+        pass
+
+    @abstractmethod
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate input data before processing.
+
+        Args:
+            input_data: Dictionary containing input data
+
+        Returns:
+            True if input is valid, False otherwise
+        """
+        pass
+
+    def get_name(self) -> str:
+        """Get the agent name."""
+        return self.name
+
+    def get_config(self) -> Dict[str, Any]:
+        """Get the agent configuration."""
+        return self.config
+
+    def log_processing(self, message: str, level: str = "info"):
+        """
+        Log processing information.
+
+        Args:
+            message: Log message
+            level: Log level (info, warning, error, debug)
+        """
+        log_method = getattr(logger, level, logger.info)
+        log_method(f"[{self.name}] {message}")
+
+    def handle_error(self, error: Exception, context: str = "") -> Dict[str, Any]:
+        """
+        Handle errors consistently across all agents.
+
+        Args:
+            error: The exception that occurred
+            context: Additional context about the error
+
+        Returns:
+            Error dictionary with details
+        """
+        error_msg = f"Error in {self.name}"
+        if context:
+            error_msg += f" ({context})"
+        error_msg += f": {str(error)}"
+
+        logger.error(error_msg)
+
+        return {
+            "success": False,
+            "error": str(error),
+            "agent": self.name,
+            "context": context
+        }
+
+    def _parse_llm_json_response(self, response_content: str) -> Dict[str, Any]:
+        """
+        Parse LLM response that may contain JSON wrapped in markdown code blocks.
+
+        Args:
+            response_content: Raw response content from LLM
+
+        Returns:
+            Parsed JSON dictionary
+
+        Raises:
+            json.JSONDecodeError: If JSON cannot be parsed
+        """
+        content = response_content.strip()
+
+        # Check if response is wrapped in markdown code block
+        if content.startswith("```json"):
+            # Remove ```json prefix and ``` suffix
+            content = content[7:]  # Remove ```json
+            if content.endswith("```"):
+                content = content[:-3]  # Remove trailing ```
+            content = content.strip()
+        elif content.startswith("```"):
+            # Remove generic ``` code block
+            content = content[3:]
+            if content.endswith("```"):
+                content = content[:-3]
+            content = content.strip()
+
+        # Parse the cleaned JSON
+        return json.loads(content)
+
+    def _safe_get(self, data: Dict[str, Any], key: str, default: Any = None) -> Any:
+        """
+        Safely get a value from a dictionary with a default.
+
+        Args:
+            data: Dictionary to get value from
+            key: Key to look up
+            default: Default value if key not found
+
+        Returns:
+            Value from dictionary or default
+        """
+        return data.get(key, default)
+
+    def _ensure_list(self, value: Any) -> list:
+        """
+        Ensure a value is a list.
+
+        Args:
+            value: Value to convert
+
+        Returns:
+            List version of value
+        """
+        if value is None:
+            return []
+        if isinstance(value, list):
+            return value
+        if isinstance(value, str):
+            # Try to parse as comma-separated
+            return [v.strip() for v in value.split(",") if v.strip()]
+        return [value]

processing_brand_sentiment/workflow/agents/comment_preprocessor_agent.py

@@ -0,0 +1,211 @@
+"""
+Comment Preprocessor Agent for brand sentiment analysis on social media comments.
+
+Extends ContentPreprocessorAgent but handles plain text (no HTML parsing).
+Builds context from content title, content description, and parent comment text
+instead of thread title and first post.
+
+Reuses: keyword sets, product alias mapping, language detection, relevance screening.
+Overrides: process() method for plain text handling and comment-specific context building.
+"""
+
+from typing import Dict, Any, Optional
+import logging
+
+from .content_preprocessor_agent import ContentPreprocessorAgent
+
+logger = logging.getLogger(__name__)
+
+
+class CommentPreprocessorAgent(ContentPreprocessorAgent):
+    """
+    Agent that preprocesses social media comments for brand sentiment analysis.
+
+    Inherits keyword detection, product alias mapping, language detection,
+    and relevance screening from ContentPreprocessorAgent.
+
+    Key differences from forum preprocessor:
+    - No HTML parsing (comments are plain text)
+    - Context built from content title + description + parent comment
+    - Different input field names (comment_text vs post_content)
+    """
+
+    def __init__(self, config: Dict[str, Any], brand_config: Dict[str, Any]):
+        """
+        Initialize the Comment Preprocessor Agent.
+
+        Args:
+            config: Agent configuration
+            brand_config: Brand-specific configuration with keywords, products, and aliases
+        """
+        super().__init__(config, brand_config)
+        self.name = "CommentPreprocessorAgent"
+
+        logger.info(
+            f"CommentPreprocessorAgent initialized with {len(self.primary_keywords)} primary keywords, "
+            f"{len(self.product_aliases)} product aliases"
+        )
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate that input contains required fields for comment processing.
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        required_fields = ["comment_sk", "comment_text"]
+        return all(field in input_data for field in required_fields)
+
+    def _build_comment_context(
+        self,
+        content_title: Optional[str] = None,
+        content_description: Optional[str] = None,
+        parent_comment_text: Optional[str] = None
+    ) -> str:
+        """
+        Build context string from social media content and parent comment information.
+
+        Args:
+            content_title: Title of the social media post/content
+            content_description: Description/message of the social media post
+            parent_comment_text: Text of the parent comment (if this is a reply)
+
+        Returns:
+            Formatted context string
+        """
+        context_parts = []
+
+        if content_title:
+            context_parts.append(f"Post title: {content_title}")
+
+        if content_description:
+            # Truncate if too long
+            truncated = content_description[:500] + "..." if len(content_description) > 500 else content_description
+            context_parts.append(f"Post description: {truncated}")
+
+        if parent_comment_text:
+            truncated = parent_comment_text[:500] + "..." if len(parent_comment_text) > 500 else parent_comment_text
+            context_parts.append(f"Parent comment: {truncated}")
+
+        return " | ".join(context_parts) if context_parts else ""
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a social media comment through the preprocessing pipeline.
+
+        Unlike forum posts, comments are plain text (no HTML parsing needed).
+        Context is built from content title, description, and parent comment.
+
+        Args:
+            input_data: Dictionary containing comment data with at least:
+                - comment_sk: Comment surrogate key
+                - comment_text: Raw comment text (plain text)
+                - content_title: Title of the post (optional)
+                - content_description: Description of the post (optional)
+                - parent_comment_text: Parent comment text if reply (optional)
+
+        Returns:
+            Dictionary with preprocessing results
+        """
+        try:
+            # Validate input
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields (comment_sk, comment_text)",
+                    **input_data
+                }
+
+            comment_text = input_data.get("comment_text", "")
+
+            # Step 1: Clean text (plain text - no HTML parsing needed)
+            cleaned_content = comment_text.strip() if comment_text else ""
+
+            # Check for empty content
+            if not cleaned_content or len(cleaned_content) < self.min_content_length:
+                return {
+                    "success": True,
+                    "cleaned_content": cleaned_content,
+                    "quoted_content": None,
+                    "is_empty": True,
+                    "preliminary_relevant": False,
+                    "needs_relevance_validation": False,
+                    **{k: v for k, v in input_data.items() if k != "comment_text"}
+                }
+
+            # Step 2: Check relevance (reused from parent class)
+            relevance_result = self._check_relevance(cleaned_content)
+            has_primary_keywords = relevance_result.get("has_primary_keywords", False)
+
+            # Step 3: Build comment context
+            raw_thread_context = self._build_comment_context(
+                content_title=input_data.get("content_title"),
+                content_description=input_data.get("content_description"),
+                parent_comment_text=input_data.get("parent_comment_text")
+            )
+
+            # Step 4: Detect language (reused from parent class)
+            lang_result = self._detect_language(cleaned_content, has_primary_keywords)
+
+            # Step 5: Extract product and competitor mentions (reused from parent class)
+            products_found = self._extract_mentioned_products(cleaned_content)
+            competitors_found = self._extract_mentioned_competitors(cleaned_content)
+
+            # Determine quoted content (parent comment serves as quoted context)
+            parent_comment = input_data.get("parent_comment_text")
+            has_parent = parent_comment is not None and str(parent_comment).strip() != ""
+
+            # Build result
+            result = {
+                "success": True,
+                "is_empty": False,
+
+                # Cleaned content
+                "cleaned_content": cleaned_content,
+                "quoted_content": parent_comment if has_parent else None,
+                "has_quote": has_parent,
+                "quoted_author": None,
+                "raw_thread_context": raw_thread_context,
+
+                # Language detection
+                "detected_language": lang_result["language"],
+                "language_code": lang_result["language_code"],
+                "is_english": lang_result["is_english"],
+                "language_confidence": lang_result["confidence"],
+                "language_detection_skipped": lang_result.get("detection_skipped", False),
+
+                # Relevance assessment
+                "preliminary_relevant": relevance_result["preliminary_relevant"],
+                "needs_relevance_validation": relevance_result["needs_relevance_validation"],
+                "relevance_keywords_found": relevance_result["found_keywords"],
+                "relevance_type": relevance_result["relevance_type"],
+                "relevance_confidence": relevance_result["relevance_confidence"],
+                "has_primary_keywords": has_primary_keywords,
+
+                # Initial extractions
+                "products_detected": products_found,
+                "competitors_detected": competitors_found,
+
+                # Preserve original data (exclude raw text to avoid duplication)
+                **{k: v for k, v in input_data.items() if k not in ["comment_text"]}
+            }
+
+            # Keep original content for reference
+            result["original_text"] = comment_text
+
+            self.log_processing(
+                f"Processed comment {input_data.get('comment_sk')}: "
+                f"lang={lang_result['language']}, "
+                f"relevant={relevance_result['preliminary_relevant']}, "
+                f"needs_validation={relevance_result['needs_relevance_validation']}, "
+                f"products={products_found}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, f"preprocessing comment {input_data.get('comment_sk')}")

processing_brand_sentiment/workflow/agents/content_preprocessor_agent.py

@@ -0,0 +1,570 @@
+"""
+Content Preprocessor Agent for brand sentiment analysis.
+Handles HTML parsing, text cleaning, language detection, product alias mapping,
+and initial relevance screening. This is a deterministic agent (no LLM calls).
+
+Enhanced version with:
+- Product alias mapping (B8 -> B8X)
+- Smart language detection (skip for short texts)
+- Always process if primary keywords found
+- Better content separation
+"""
+
+import re
+from typing import Dict, Any, List, Optional, Set
+from lingua import Language, LanguageDetectorBuilder
+import logging
+
+from .base_agent import BaseAgent
+from utils.html_parser import HTMLParser
+
+logger = logging.getLogger(__name__)
+
+
+class ContentPreprocessorAgent(BaseAgent):
+    """
+    Agent that preprocesses forum posts:
+    - Parses HTML to extract reply and quoted content
+    - Cleans and normalizes text
+    - Maps product aliases to canonical names
+    - Detects language (with smart handling for short texts)
+    - Performs initial keyword-based relevance screening
+    """
+
+    # Lingua to ISO 639-1 language code mapping
+    LINGUA_TO_ISO = {
+        Language.ENGLISH: "en",
+        Language.SPANISH: "es",
+        Language.FRENCH: "fr",
+        Language.GERMAN: "de",
+        Language.ITALIAN: "it",
+        Language.PORTUGUESE: "pt",
+        Language.RUSSIAN: "ru",
+        Language.JAPANESE: "ja",
+        Language.KOREAN: "ko",
+        Language.CHINESE: "zh",
+        Language.ARABIC: "ar",
+        Language.HINDI: "hi",
+        Language.DUTCH: "nl",
+        Language.SWEDISH: "sv",
+        Language.POLISH: "pl",
+        Language.TURKISH: "tr"
+    }
+
+    def __init__(self, config: Dict[str, Any], brand_config: Dict[str, Any]):
+        """
+        Initialize the Content Preprocessor Agent.
+
+        Args:
+            config: Agent configuration
+            brand_config: Brand-specific configuration with keywords, products, and aliases
+        """
+        super().__init__("ContentPreprocessorAgent", config)
+        self.brand_config = brand_config
+        self.html_parser = HTMLParser()
+
+        # Get preprocessing settings
+        preprocessing_config = brand_config.get("preprocessing", {})
+        self.min_length_for_lang_detection = preprocessing_config.get(
+            "min_length_for_language_detection", 50
+        )
+        self.default_language = preprocessing_config.get(
+            "default_language_for_short_text", "English"
+        )
+        self.always_process_primary = preprocessing_config.get(
+            "always_process_if_primary_keyword", True
+        )
+        self.min_content_length = preprocessing_config.get("min_content_length", 3)
+
+        # Initialize lingua detector
+        self.language_detector = LanguageDetectorBuilder.from_all_languages().build()
+
+        # Build keyword sets and alias mappings
+        self._build_keyword_sets()
+        self._build_alias_mappings()
+
+        logger.info(
+            f"ContentPreprocessorAgent initialized with {len(self.primary_keywords)} primary keywords, "
+            f"{len(self.product_aliases)} product aliases"
+        )
+
+    def _build_keyword_sets(self) -> None:
+        """Build keyword sets from brand configuration for efficient relevance checking."""
+        relevance_config = self.brand_config.get("relevance_keywords", {})
+
+        # Primary keywords - definitive Sabian mentions
+        primary = relevance_config.get("primary", {}).get("keywords", [])
+        self.primary_keywords: Set[str] = set(k.lower() for k in primary)
+
+        # Contextual keywords - need disambiguation (HH, AA)
+        contextual = relevance_config.get("contextual", {}).get("keywords", [])
+        self.contextual_keywords: Set[str] = set(k.lower() for k in contextual)
+
+        # Cymbal context keywords - help disambiguate contextual terms
+        cymbal_context = relevance_config.get("cymbal_context", {}).get("keywords", [])
+        self.cymbal_context_keywords: Set[str] = set(k.lower() for k in cymbal_context)
+
+        # Competitor names and aliases for detection
+        competitors = self.brand_config.get("brand", {}).get("competitors", [])
+        self.competitor_keywords: Set[str] = set()
+        self.competitor_name_map: Dict[str, str] = {}  # alias -> canonical name
+
+        for comp in competitors:
+            if isinstance(comp, dict):
+                name = comp.get("name", "")
+                self.competitor_keywords.add(name.lower())
+                self.competitor_name_map[name.lower()] = name
+                for alias in comp.get("aliases", []):
+                    alias_lower = alias.lower()
+                    self.competitor_keywords.add(alias_lower)
+                    self.competitor_name_map[alias_lower] = name
+            else:
+                comp_str = str(comp).lower()
+                self.competitor_keywords.add(comp_str)
+                self.competitor_name_map[comp_str] = str(comp)
+
+        # Product names
+        products = self.brand_config.get("brand", {}).get("products", [])
+        self.product_keywords: Set[str] = set(p.lower() for p in products)
+        self.products_list = products  # Keep original case
+
+        logger.debug(
+            f"Built keyword sets: {len(self.primary_keywords)} primary, "
+            f"{len(self.contextual_keywords)} contextual, "
+            f"{len(self.product_keywords)} products, "
+            f"{len(self.competitor_keywords)} competitor terms"
+        )
+
+    def _build_alias_mappings(self) -> None:
+        """Build product alias mappings from brand configuration."""
+        aliases = self.brand_config.get("brand", {}).get("product_aliases", {})
+
+        # Build alias -> canonical product mapping
+        self.product_aliases: Dict[str, str] = {}
+        for alias, canonical in aliases.items():
+            self.product_aliases[alias.lower()] = canonical
+
+        # Also add primary keywords that are aliases to contextual keywords
+        # e.g., "b8" should trigger contextual check since it maps to "B8X"
+        for alias in self.product_aliases.keys():
+            if alias not in self.primary_keywords:
+                self.contextual_keywords.add(alias)
+
+        logger.debug(f"Built {len(self.product_aliases)} product alias mappings")
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate that input contains required fields.
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        required_fields = ["post_id", "post_content"]
+        return all(field in input_data for field in required_fields)
+
+    def _detect_language(self, text: str, has_primary_keywords: bool = False) -> Dict[str, Any]:
+        """
+        Detect the language of text using lingua library.
+
+        Enhanced logic:
+        - Skip detection for short texts (< min_length_for_lang_detection chars)
+        - Always return English if primary Sabian keywords are found
+
+        Args:
+            text: Text to analyze
+            has_primary_keywords: Whether primary Sabian keywords were found
+
+        Returns:
+            Dictionary with language detection results
+        """
+        try:
+            cleaned_text = text.strip()
+
+            # If text is too short, default to English
+            if len(cleaned_text) < self.min_length_for_lang_detection:
+                return {
+                    "language": self.default_language,
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "low",
+                    "detection_skipped": True,
+                    "skip_reason": f"Text too short ({len(cleaned_text)} < {self.min_length_for_lang_detection} chars)"
+                }
+
+            # If primary keywords found and always_process_primary is True, treat as English
+            if has_primary_keywords and self.always_process_primary:
+                # Still try to detect, but override if non-English
+                detected = self.language_detector.detect_language_of(cleaned_text)
+
+                if detected == Language.ENGLISH:
+                    return {
+                        "language": "English",
+                        "language_code": "en",
+                        "is_english": True,
+                        "confidence": "high",
+                        "detection_skipped": False,
+                        "skip_reason": None
+                    }
+                else:
+                    # Primary keyword found but detected as non-English
+                    # Force to English since Sabian is explicitly mentioned
+                    lang_name = detected.name.capitalize() if detected else "Unknown"
+                    return {
+                        "language": "English",
+                        "language_code": "en",
+                        "is_english": True,
+                        "confidence": "medium",
+                        "detection_skipped": False,
+                        "skip_reason": None,
+                        "original_detected_language": lang_name,
+                        "override_reason": "Primary Sabian keyword found, treating as English"
+                    }
+
+            # Standard detection
+            detected = self.language_detector.detect_language_of(cleaned_text)
+
+            if detected is None:
+                return {
+                    "language": self.default_language,
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "low",
+                    "detection_skipped": False,
+                    "skip_reason": None
+                }
+
+            if detected == Language.ENGLISH:
+                return {
+                    "language": "English",
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "high",
+                    "detection_skipped": False,
+                    "skip_reason": None
+                }
+
+            lang_code = self.LINGUA_TO_ISO.get(detected, "unknown")
+            lang_name = detected.name.capitalize()
+
+            return {
+                "language": lang_name,
+                "language_code": lang_code,
+                "is_english": False,
+                "confidence": "high",
+                "detection_skipped": False,
+                "skip_reason": None
+            }
+
+        except Exception as e:
+            logger.warning(f"Language detection failed: {e}")
+            return {
+                "language": self.default_language,
+                "language_code": "en",
+                "is_english": True,
+                "confidence": "low",
+                "detection_skipped": False,
+                "skip_reason": None,
+                "detection_error": str(e)
+            }
+
+    def _normalize_product_mentions(self, found_products: List[str]) -> List[str]:
+        """
+        Normalize product mentions using alias mappings.
+
+        Args:
+            found_products: List of product terms found
+
+        Returns:
+            List of canonical product names
+        """
+        normalized = []
+        for product in found_products:
+            product_lower = product.lower()
+
+            # Check if it's an alias
+            if product_lower in self.product_aliases:
+                canonical = self.product_aliases[product_lower]
+                if canonical not in normalized:
+                    normalized.append(canonical)
+            # Check if it's a direct product match
+            elif product_lower in self.product_keywords:
+                # Find the original case version
+                for p in self.products_list:
+                    if p.lower() == product_lower:
+                        if p not in normalized:
+                            normalized.append(p)
+                        break
+
+        return normalized
+
+    def _check_relevance(self, text: str) -> Dict[str, Any]:
+        """
+        Check if text is relevant to the brand using keyword matching.
+
+        Enhanced to handle product aliases.
+
+        Returns:
+            Dictionary with relevance assessment
+        """
+        text_lower = text.lower()
+
+        # Tokenize for word boundary matching
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        # Also check for multi-word phrases (for aliases like "hand hammered")
+        all_aliases = set(self.product_aliases.keys())
+
+        # Check for primary keywords (definitive matches)
+        found_primary = self.primary_keywords.intersection(words)
+
+        # Check for product aliases in text
+        found_aliases = []
+        for alias in all_aliases:
+            if ' ' in alias:
+                # Multi-word alias - check in full text
+                if alias in text_lower:
+                    found_aliases.append(alias)
+            elif alias in words:
+                found_aliases.append(alias)
+
+        # Map aliases to canonical products
+        alias_products = []
+        for alias in found_aliases:
+            if alias in self.product_aliases:
+                canonical = self.product_aliases[alias]
+                if canonical not in alias_products:
+                    alias_products.append(canonical)
+
+        if found_primary or alias_products:
+            all_found = list(found_primary) + found_aliases
+            return {
+                "preliminary_relevant": True,
+                "needs_relevance_validation": False,
+                "found_keywords": all_found,
+                "mapped_products": alias_products,
+                "relevance_type": "primary",
+                "relevance_confidence": "high",
+                "has_primary_keywords": True
+            }
+
+        # Check for contextual keywords (need validation)
+        found_contextual = self.contextual_keywords.intersection(words)
+        if found_contextual:
+            # Check if there's cymbal context
+            found_cymbal_context = self.cymbal_context_keywords.intersection(words)
+            has_cymbal_context = len(found_cymbal_context) > 0
+
+            return {
+                "preliminary_relevant": True,
+                "needs_relevance_validation": True,
+                "found_keywords": list(found_contextual),
+                "cymbal_context_found": list(found_cymbal_context) if found_cymbal_context else [],
+                "has_cymbal_context": has_cymbal_context,
+                "mapped_products": [],
+                "relevance_type": "contextual",
+                "relevance_confidence": "medium" if has_cymbal_context else "low",
+                "has_primary_keywords": False
+            }
+
+        # Check for competitor mentions (might be comparative discussion)
+        found_competitors = self.competitor_keywords.intersection(words)
+        if found_competitors:
+            return {
+                "preliminary_relevant": False,
+                "needs_relevance_validation": True,
+                "found_keywords": list(found_competitors),
+                "mapped_products": [],
+                "relevance_type": "competitor_only",
+                "relevance_confidence": "low",
+                "has_primary_keywords": False
+            }
+
+        # No relevant keywords found
+        return {
+            "preliminary_relevant": False,
+            "needs_relevance_validation": False,
+            "found_keywords": [],
+            "mapped_products": [],
+            "relevance_type": "none",
+            "relevance_confidence": "high",
+            "has_primary_keywords": False
+        }
+
+    def _extract_mentioned_products(self, text: str) -> List[str]:
+        """
+        Extract product names mentioned in the text, including aliases.
+
+        Args:
+            text: Text to search
+
+        Returns:
+            List of canonical product names found
+        """
+        text_lower = text.lower()
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        found_products = []
+
+        # Check direct product mentions
+        for product in self.products_list:
+            if product.lower() in words:
+                if product not in found_products:
+                    found_products.append(product)
+
+        # Check aliases
+        for alias, canonical in self.product_aliases.items():
+            if ' ' in alias:
+                # Multi-word alias
+                if alias in text_lower:
+                    if canonical not in found_products:
+                        found_products.append(canonical)
+            elif alias in words:
+                if canonical not in found_products:
+                    found_products.append(canonical)
+
+        return found_products
+
+    def _extract_mentioned_competitors(self, text: str) -> List[str]:
+        """
+        Extract competitor brand names mentioned in the text.
+
+        Args:
+            text: Text to search
+
+        Returns:
+            List of canonical competitor names found
+        """
+        text_lower = text.lower()
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        found_competitors = set()
+
+        for alias in self.competitor_keywords:
+            if ' ' in alias:
+                # Multi-word check
+                if alias in text_lower:
+                    canonical = self.competitor_name_map.get(alias, alias)
+                    found_competitors.add(canonical)
+            elif alias in words:
+                canonical = self.competitor_name_map.get(alias, alias)
+                found_competitors.add(canonical)
+
+        return list(found_competitors)
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a forum post through preprocessing pipeline.
+
+        Args:
+            input_data: Dictionary containing post data with at least:
+                - post_id: Post identifier
+                - post_content: Raw HTML content
+                - thread_title: Thread title (optional)
+                - thread_first_post: First post content (optional)
+                - category_title: Category title (optional)
+                - category_topic: Category topic (optional)
+
+        Returns:
+            Dictionary with preprocessing results
+        """
+        try:
+            # Validate input
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    **input_data
+                }
+
+            post_content = input_data.get("post_content", "")
+
+            # Step 1: Parse HTML content
+            parsed = self.html_parser.parse_post_content(post_content)
+            reply_content = parsed.get("reply_content", "")
+            quoted_content = parsed.get("quoted_content")
+
+            # Check for empty content
+            if not reply_content or len(reply_content.strip()) < self.min_content_length:
+                return {
+                    "success": True,
+                    "cleaned_content": reply_content,
+                    "quoted_content": quoted_content,
+                    "is_empty": True,
+                    "preliminary_relevant": False,
+                    "needs_relevance_validation": False,
+                    **{k: v for k, v in input_data.items() if k != "post_content"}
+                }
+
+            # Step 2: Check relevance FIRST (needed for language detection logic)
+            relevance_result = self._check_relevance(reply_content)
+            has_primary_keywords = relevance_result.get("has_primary_keywords", False)
+
+            # Step 3: Build thread context (raw - will be summarized by extraction agent)
+            raw_thread_context = self.html_parser.build_thread_context(
+                thread_title=input_data.get("thread_title"),
+                first_post_content=input_data.get("thread_first_post"),
+                category_title=input_data.get("category_title"),
+                category_topic=input_data.get("category_topic")
+            )
+
+            # Step 4: Detect language (with smart handling)
+            lang_result = self._detect_language(reply_content, has_primary_keywords)
+
+            # Step 5: Extract product and competitor mentions from actual post content
+            products_found = self._extract_mentioned_products(reply_content)
+            competitors_found = self._extract_mentioned_competitors(reply_content)
+
+            # Build result
+            result = {
+                "success": True,
+                "is_empty": False,
+
+                # Cleaned content
+                "cleaned_content": reply_content,
+                "quoted_content": quoted_content,
+                "has_quote": parsed.get("has_quote", False),
+                "quoted_author": parsed.get("quoted_author"),
+                "raw_thread_context": raw_thread_context,
+
+                # Language detection
+                "detected_language": lang_result["language"],
+                "language_code": lang_result["language_code"],
+                "is_english": lang_result["is_english"],
+                "language_confidence": lang_result["confidence"],
+                "language_detection_skipped": lang_result.get("detection_skipped", False),
+
+                # Relevance assessment
+                "preliminary_relevant": relevance_result["preliminary_relevant"],
+                "needs_relevance_validation": relevance_result["needs_relevance_validation"],
+                "relevance_keywords_found": relevance_result["found_keywords"],
+                "relevance_type": relevance_result["relevance_type"],
+                "relevance_confidence": relevance_result["relevance_confidence"],
+                "has_primary_keywords": has_primary_keywords,
+
+                # Initial extractions
+                "products_detected": products_found,
+                "competitors_detected": competitors_found,
+
+                # Preserve original data
+                **{k: v for k, v in input_data.items() if k not in ["post_content"]}
+            }
+
+            # Keep original content for reference
+            result["original_content"] = post_content
+
+            self.log_processing(
+                f"Processed post {input_data.get('post_id')}: "
+                f"lang={lang_result['language']}, "
+                f"relevant={relevance_result['preliminary_relevant']}, "
+                f"needs_validation={relevance_result['needs_relevance_validation']}, "
+                f"products={products_found}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, f"preprocessing post {input_data.get('post_id')}")

processing_brand_sentiment/workflow/agents/output_validator_agent.py

@@ -0,0 +1,408 @@
+"""
+Output Validator Agent for brand sentiment analysis.
+
+This agent performs rule-based validation on the final output to ensure:
+1. All values are from predefined lists
+2. Logical consistency between fields
+3. Anomaly detection for manual review flagging
+
+This is a deterministic agent (no LLM calls) that acts as a quality gate.
+"""
+
+from typing import Dict, Any, List, Set
+import logging
+
+from .base_agent import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class OutputValidatorAgent(BaseAgent):
+    """
+    Agent that validates the final output for consistency and quality.
+
+    Performs rule-based checks without LLM calls to ensure data quality
+    and flag posts that may need manual review.
+    """
+
+    def __init__(
+        self,
+        config: Dict[str, Any],
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any]
+    ):
+        """
+        Initialize the Output Validator Agent.
+
+        Args:
+            config: Agent configuration
+            brand_config: Brand-specific configuration
+            analysis_categories: Category definitions for validation
+        """
+        super().__init__("OutputValidatorAgent", config)
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+
+        # Build valid value sets for validation
+        self._build_valid_value_sets()
+
+        logger.info("OutputValidatorAgent initialized")
+
+    def _build_valid_value_sets(self) -> None:
+        """Build sets of valid values for efficient validation."""
+        brand = self.brand_config.get("brand", {})
+
+        # Products
+        self.valid_products: Set[str] = set(
+            p.lower() for p in brand.get("products", [])
+        )
+        self.products_canonical = {p.lower(): p for p in brand.get("products", [])}
+
+        # Competitors
+        self.valid_competitors: Set[str] = set()
+        self.competitors_canonical = {}
+        for comp in brand.get("competitors", []):
+            if isinstance(comp, dict):
+                name = comp.get("name", "")
+                self.valid_competitors.add(name.lower())
+                self.competitors_canonical[name.lower()] = name
+
+        # Extract all category values
+        self.valid_values = {}
+
+        category_configs = {
+            "author_role": self.analysis_categories.get("author_role", {}),
+            "sabian_mention_context": self.analysis_categories.get("sabian_mention_context", {}),
+            "sentiment_level": self.analysis_categories.get("sentiment", {}),
+            "emotion_type": self.analysis_categories.get("emotions", {}),
+            "intents": self.analysis_categories.get("intents", {}),
+            "purchase_stage": self.analysis_categories.get("purchase_stage", {}),
+            "comparison_type": self.analysis_categories.get("comparison_type", {}),
+            "feedback_aspects": self.analysis_categories.get("feedback_aspects", {}),
+            "decision_drivers": self.analysis_categories.get("decision_drivers", {}),
+            "product_attributes": self.analysis_categories.get("product_attributes", {}),
+        }
+
+        for key, config in category_configs.items():
+            if "categories" in config:
+                self.valid_values[key] = set(
+                    c["value"].lower() for c in config["categories"]
+                )
+            elif "levels" in config:
+                self.valid_values[key] = set(
+                    c["value"].lower() for c in config["levels"]
+                )
+            else:
+                self.valid_values[key] = set()
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """Validate that input contains required fields."""
+        # The validator accepts any input - it will validate what's there
+        return True
+
+    def _validate_list_values(
+        self,
+        values: List[Any],
+        valid_set: Set[str],
+        field_name: str
+    ) -> Dict[str, Any]:
+        """
+        Validate list values against a set of valid values.
+
+        Returns:
+            Dictionary with validation results
+        """
+        if not values:
+            return {"valid": True, "invalid_values": [], "field": field_name}
+
+        invalid = []
+        for v in values:
+            if isinstance(v, str) and v.lower() not in valid_set:
+                invalid.append(v)
+
+        return {
+            "valid": len(invalid) == 0,
+            "invalid_values": invalid,
+            "field": field_name
+        }
+
+    def _validate_single_value(
+        self,
+        value: Any,
+        valid_set: Set[str],
+        field_name: str,
+        allow_none: bool = True
+    ) -> Dict[str, Any]:
+        """
+        Validate a single value against a set of valid values.
+
+        Returns:
+            Dictionary with validation results
+        """
+        if value is None:
+            return {"valid": allow_none, "invalid_value": None if allow_none else value, "field": field_name}
+
+        if isinstance(value, str) and value.lower() in valid_set:
+            return {"valid": True, "invalid_value": None, "field": field_name}
+
+        return {"valid": False, "invalid_value": value, "field": field_name}
+
+    def _check_logical_consistency(self, data: Dict[str, Any]) -> List[str]:
+        """
+        Check for logical consistency between fields.
+
+        Note: Empty products_mentioned is OK even when relevant - users may
+        discuss the Sabian brand generally without specific products.
+
+        Returns:
+            List of inconsistency warnings
+        """
+        warnings = []
+        is_relevant = data.get("is_relevant", False)
+
+        # Check 1: If not relevant, certain fields should be empty/null
+        if not is_relevant:
+            if data.get("sabian_mention_context"):
+                warnings.append(
+                    "sabian_mention_context should be null when is_relevant=False"
+                )
+            if data.get("sentiment_level") and data.get("sentiment_level") != "neutral":
+                warnings.append(
+                    "sentiment_level should be null/neutral when is_relevant=False"
+                )
+
+        # Check 2: Comparison type should only be set if comparing intent exists
+        if data.get("comparison_type"):
+            intents = data.get("intents", [])
+            if "comparing" not in intents:
+                warnings.append(
+                    "comparison_type is set but 'comparing' not in intents"
+                )
+
+        # Check 3: Author perspective fields consistency
+        # If author is giving advice (providing_information) without sharing experience,
+        # pain_points and delight_factors should typically be empty
+        intents = data.get("intents", [])
+        if "providing_information" in intents and "sharing_experience" not in intents:
+            if data.get("pain_points") or data.get("delight_factors"):
+                warnings.append(
+                    "pain_points/delight_factors set for advice-giving post without sharing_experience intent"
+                )
+
+        return warnings
+
+    def _fix_overlapping_feedback(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Fix overlapping values between pain_points and delight_factors.
+
+        Rule: The same aspect cannot be both a pain point and a delight factor.
+        Resolution: Use sentiment to determine which to keep, or clear both if neutral.
+
+        Args:
+            data: Dictionary with analysis results
+
+        Returns:
+            Updated dictionary with fixed pain_points and delight_factors
+        """
+        pain_points = data.get("pain_points", []) or []
+        delight_factors = data.get("delight_factors", []) or []
+
+        if not pain_points or not delight_factors:
+            return data
+
+        # Find overlapping values
+        pain_set = set(p.lower() if isinstance(p, str) else p for p in pain_points)
+        delight_set = set(d.lower() if isinstance(d, str) else d for d in delight_factors)
+        overlap = pain_set.intersection(delight_set)
+
+        if not overlap:
+            return data
+
+        # Get sentiment to determine which to keep
+        sentiment = data.get("sentiment_level", "neutral")
+
+        # Create new lists without overlapping values
+        if sentiment in ["positive", "very_positive"]:
+            # Keep in delight_factors, remove from pain_points
+            new_pain_points = [p for p in pain_points if p.lower() not in overlap]
+            new_delight_factors = delight_factors
+        elif sentiment in ["negative", "very_negative"]:
+            # Keep in pain_points, remove from delight_factors
+            new_pain_points = pain_points
+            new_delight_factors = [d for d in delight_factors if d.lower() not in overlap]
+        else:
+            # Neutral sentiment - clear both (can't determine intent)
+            new_pain_points = [p for p in pain_points if p.lower() not in overlap]
+            new_delight_factors = [d for d in delight_factors if d.lower() not in overlap]
+
+        # Update data
+        data["pain_points"] = new_pain_points
+        data["delight_factors"] = new_delight_factors
+
+        logger.debug(
+            f"Fixed overlapping feedback: removed {overlap} from "
+            f"{'pain_points' if sentiment in ['positive', 'very_positive'] else 'delight_factors' if sentiment in ['negative', 'very_negative'] else 'both'}"
+        )
+
+        return data
+
+    def _detect_anomalies(self, data: Dict[str, Any]) -> List[str]:
+        """
+        Detect anomalies that might need manual review.
+
+        Returns:
+            List of anomaly flags
+        """
+        anomalies = []
+
+        # Anomaly 1: Low confidence relevance
+        if data.get("is_relevant") and data.get("relevance_confidence") == "low":
+            anomalies.append("low_confidence_relevant")
+
+        # Anomaly 2: Sarcasm detected - sentiment might be inverted
+        if data.get("sarcasm_detected"):
+            anomalies.append("sarcasm_detected")
+
+        # Anomaly 3: Very short content marked as relevant
+        content = data.get("cleaned_content", "")
+        if data.get("is_relevant") and len(content) < 20:
+            anomalies.append("short_relevant_content")
+
+        # Anomaly 4: Switching behavior detected
+        comparison_type = data.get("comparison_type", "")
+        if comparison_type in ["switching_to_sabian", "switching_from_sabian"]:
+            anomalies.append(f"brand_switching_{comparison_type}")
+
+        return anomalies
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process and validate the analysis output.
+
+        Args:
+            input_data: Dictionary with all analysis results
+
+        Returns:
+            Dictionary with validation results added
+        """
+        try:
+            validation_errors = []
+            validation_warnings = []
+
+            # Skip detailed validation for non-relevant or skipped posts
+            if not input_data.get("is_relevant", False) or input_data.get("analysis_skipped", False):
+                return {
+                    **input_data,
+                    "validation_passed": True,
+                    "validation_errors": [],
+                    "validation_warnings": [],
+                    "validation_flags": [],
+                    "processing_status": "completed"
+                }
+
+            # Fix overlapping pain_points and delight_factors (safety net)
+            input_data = self._fix_overlapping_feedback(input_data)
+
+            # Validate products_mentioned
+            products_result = self._validate_list_values(
+                input_data.get("products_mentioned", []),
+                self.valid_products,
+                "products_mentioned"
+            )
+            if not products_result["valid"]:
+                validation_errors.append(
+                    f"Invalid products: {products_result['invalid_values']}"
+                )
+
+            # Validate competitors_mentioned
+            competitors_result = self._validate_list_values(
+                input_data.get("competitors_mentioned", []),
+                self.valid_competitors,
+                "competitors_mentioned"
+            )
+            if not competitors_result["valid"]:
+                validation_errors.append(
+                    f"Invalid competitors: {competitors_result['invalid_values']}"
+                )
+
+            # Validate categorical fields
+            categorical_validations = [
+                ("author_role", "author_role", True),
+                ("sabian_mention_context", "sabian_mention_context", True),
+                ("sentiment_level", "sentiment_level", True),
+                ("emotion_type", "emotion_type", True),
+                ("purchase_stage", "purchase_stage", True),
+                ("comparison_type", "comparison_type", True),
+            ]
+
+            for field, valid_key, allow_none in categorical_validations:
+                result = self._validate_single_value(
+                    input_data.get(field),
+                    self.valid_values.get(valid_key, set()),
+                    field,
+                    allow_none
+                )
+                if not result["valid"]:
+                    validation_errors.append(
+                        f"Invalid {field}: {result['invalid_value']}"
+                    )
+
+            # Validate list fields
+            list_validations = [
+                ("intents", "intents"),
+                ("product_attributes", "product_attributes"),
+                ("pain_points", "feedback_aspects"),
+                ("delight_factors", "feedback_aspects"),
+                ("decision_drivers", "decision_drivers"),
+            ]
+
+            for field, valid_key in list_validations:
+                result = self._validate_list_values(
+                    input_data.get(field, []),
+                    self.valid_values.get(valid_key, set()),
+                    field
+                )
+                if not result["valid"]:
+                    validation_warnings.append(
+                        f"Invalid values in {field}: {result['invalid_values']}"
+                    )
+
+            # Check logical consistency
+            consistency_warnings = self._check_logical_consistency(input_data)
+            validation_warnings.extend(consistency_warnings)
+
+            # Detect anomalies
+            anomalies = self._detect_anomalies(input_data)
+
+            # Determine overall validation status
+            validation_passed = len(validation_errors) == 0
+
+            # Set processing status
+            if validation_errors:
+                processing_status = "validation_failed"
+            elif anomalies:
+                processing_status = "completed_with_flags"
+            else:
+                processing_status = "completed"
+
+            result = {
+                **input_data,
+                "validation_passed": validation_passed,
+                "validation_errors": validation_errors,
+                "validation_warnings": validation_warnings,
+                "validation_flags": anomalies,
+                "processing_status": processing_status
+            }
+
+            if validation_errors or validation_warnings or anomalies:
+                self.log_processing(
+                    f"Validation complete: passed={validation_passed}, "
+                    f"errors={len(validation_errors)}, warnings={len(validation_warnings)}, "
+                    f"flags={anomalies}",
+                    "debug"
+                )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "output validation")

processing_brand_sentiment/workflow/agents/preprocessor_agent.py

@@ -0,0 +1,408 @@
+"""
+Preprocessor Agent for brand sentiment analysis.
+Handles HTML parsing, text cleaning, language detection, and initial relevance screening.
+This is a deterministic agent (no LLM calls except for language detection fallback).
+"""
+
+import re
+from typing import Dict, Any, List, Optional, Set
+from lingua import Language, LanguageDetectorBuilder
+import logging
+
+from .base_agent import BaseAgent
+from utils.html_parser import HTMLParser
+
+logger = logging.getLogger(__name__)
+
+
+class PreprocessorAgent(BaseAgent):
+    """
+    Agent that preprocesses forum posts:
+    - Parses HTML to extract reply and quoted content
+    - Cleans and normalizes text
+    - Detects language
+    - Performs initial keyword-based relevance screening
+    """
+
+    # Lingua to ISO 639-1 language code mapping
+    LINGUA_TO_ISO = {
+        Language.ENGLISH: "en",
+        Language.SPANISH: "es",
+        Language.FRENCH: "fr",
+        Language.GERMAN: "de",
+        Language.ITALIAN: "it",
+        Language.PORTUGUESE: "pt",
+        Language.RUSSIAN: "ru",
+        Language.JAPANESE: "ja",
+        Language.KOREAN: "ko",
+        Language.CHINESE: "zh",
+        Language.ARABIC: "ar",
+        Language.HINDI: "hi",
+        Language.DUTCH: "nl",
+        Language.SWEDISH: "sv",
+        Language.POLISH: "pl",
+        Language.TURKISH: "tr"
+    }
+
+    def __init__(self, config: Dict[str, Any], brand_config: Dict[str, Any]):
+        """
+        Initialize the Preprocessor Agent.
+
+        Args:
+            config: Agent configuration
+            brand_config: Brand-specific configuration with keywords and products
+        """
+        super().__init__("PreprocessorAgent", config)
+        self.brand_config = brand_config
+        self.html_parser = HTMLParser()
+
+        # Initialize lingua detector
+        self.language_detector = LanguageDetectorBuilder.from_all_languages().build()
+
+        # Build keyword sets for efficient lookup
+        self._build_keyword_sets()
+
+        logger.info("PreprocessorAgent initialized")
+
+    def _build_keyword_sets(self) -> None:
+        """Build keyword sets from brand configuration for efficient relevance checking."""
+        relevance_config = self.brand_config.get("relevance_keywords", {})
+
+        # Primary keywords - definitive Sabian mentions
+        primary = relevance_config.get("primary", {}).get("keywords", [])
+        self.primary_keywords: Set[str] = set(k.lower() for k in primary)
+
+        # Contextual keywords - need disambiguation (HH, AA)
+        contextual = relevance_config.get("contextual", {}).get("keywords", [])
+        self.contextual_keywords: Set[str] = set(k.lower() for k in contextual)
+
+        # Cymbal context keywords - help disambiguate contextual terms
+        cymbal_context = relevance_config.get("cymbal_context", {}).get("keywords", [])
+        self.cymbal_context_keywords: Set[str] = set(k.lower() for k in cymbal_context)
+
+        # Competitor names for detection
+        competitors = self.brand_config.get("brand", {}).get("competitors", [])
+        self.competitor_keywords: Set[str] = set()
+        for comp in competitors:
+            if isinstance(comp, dict):
+                self.competitor_keywords.add(comp.get("name", "").lower())
+                for alias in comp.get("aliases", []):
+                    self.competitor_keywords.add(alias.lower())
+            else:
+                self.competitor_keywords.add(str(comp).lower())
+
+        # Product names
+        products = self.brand_config.get("brand", {}).get("products", [])
+        self.product_keywords: Set[str] = set(p.lower() for p in products)
+
+        logger.info(f"Built keyword sets: {len(self.primary_keywords)} primary, "
+                   f"{len(self.contextual_keywords)} contextual, "
+                   f"{len(self.product_keywords)} products")
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate that input contains required fields.
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        required_fields = ["post_id", "post_content"]
+        return all(field in input_data for field in required_fields)
+
+    def _detect_language(self, text: str) -> Dict[str, Any]:
+        """
+        Detect the language of text using lingua library.
+
+        Args:
+            text: Text to analyze
+
+        Returns:
+            Dictionary with language detection results
+        """
+        try:
+            cleaned_text = text.strip()
+            if not cleaned_text or len(cleaned_text) < 3:
+                return {
+                    "language": "English",
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "low"
+                }
+
+            detected = self.language_detector.detect_language_of(cleaned_text)
+
+            if detected is None:
+                return {
+                    "language": "English",
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "low"
+                }
+
+            if detected == Language.ENGLISH:
+                return {
+                    "language": "English",
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "high"
+                }
+
+            lang_code = self.LINGUA_TO_ISO.get(detected, "unknown")
+            lang_name = detected.name.capitalize()
+
+            return {
+                "language": lang_name,
+                "language_code": lang_code,
+                "is_english": False,
+                "confidence": "high"
+            }
+
+        except Exception as e:
+            logger.warning(f"Language detection failed: {e}")
+            return {
+                "language": "English",
+                "language_code": "en",
+                "is_english": True,
+                "confidence": "low"
+            }
+
+    def _check_relevance(self, text: str) -> Dict[str, Any]:
+        """
+        Check if text is relevant to the brand using keyword matching.
+
+        Returns:
+            Dictionary with relevance assessment:
+            - preliminary_relevant: Initial relevance assessment
+            - needs_relevance_validation: True if contains ambiguous terms needing LLM check
+            - found_keywords: Keywords found in the text
+            - relevance_type: 'primary', 'contextual', or 'none'
+        """
+        text_lower = text.lower()
+
+        # Tokenize for word boundary matching
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        # Check for primary keywords (definitive matches)
+        found_primary = self.primary_keywords.intersection(words)
+        if found_primary:
+            return {
+                "preliminary_relevant": True,
+                "needs_relevance_validation": False,
+                "found_keywords": list(found_primary),
+                "relevance_type": "primary",
+                "relevance_confidence": "high"
+            }
+
+        # Check for contextual keywords (need validation)
+        found_contextual = self.contextual_keywords.intersection(words)
+        if found_contextual:
+            # Check if there's cymbal context
+            found_cymbal_context = self.cymbal_context_keywords.intersection(words)
+            has_cymbal_context = len(found_cymbal_context) > 0
+
+            return {
+                "preliminary_relevant": True,  # Potentially relevant
+                "needs_relevance_validation": True,  # Needs LLM confirmation
+                "found_keywords": list(found_contextual),
+                "cymbal_context_found": list(found_cymbal_context) if found_cymbal_context else [],
+                "has_cymbal_context": has_cymbal_context,
+                "relevance_type": "contextual",
+                "relevance_confidence": "medium" if has_cymbal_context else "low"
+            }
+
+        # Check for competitor mentions (might be comparative discussion)
+        found_competitors = self.competitor_keywords.intersection(words)
+        if found_competitors:
+            # Has competitor mention but no Sabian mention
+            # Could still be relevant in a comparison context
+            return {
+                "preliminary_relevant": False,
+                "needs_relevance_validation": True,  # LLM should check context
+                "found_keywords": list(found_competitors),
+                "relevance_type": "competitor_only",
+                "relevance_confidence": "low"
+            }
+
+        # No relevant keywords found
+        return {
+            "preliminary_relevant": False,
+            "needs_relevance_validation": False,
+            "found_keywords": [],
+            "relevance_type": "none",
+            "relevance_confidence": "high"
+        }
+
+    def _extract_mentioned_products(self, text: str) -> List[str]:
+        """
+        Extract product names mentioned in the text.
+
+        Args:
+            text: Text to search
+
+        Returns:
+            List of product names found
+        """
+        text_lower = text.lower()
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        found_products = []
+        products = self.brand_config.get("brand", {}).get("products", [])
+
+        for product in products:
+            if product.lower() in words:
+                found_products.append(product)
+
+        return found_products
+
+    def _extract_mentioned_competitors(self, text: str) -> List[str]:
+        """
+        Extract competitor names mentioned in the text.
+
+        Args:
+            text: Text to search
+
+        Returns:
+            List of competitor names found
+        """
+        text_lower = text.lower()
+        words = set(re.findall(r'\b[a-zA-Z0-9]+\b', text_lower))
+
+        found_competitors = []
+        competitors = self.brand_config.get("brand", {}).get("competitors", [])
+
+        for comp in competitors:
+            if isinstance(comp, dict):
+                name = comp.get("name", "")
+                aliases = comp.get("aliases", [])
+
+                # Check name and aliases
+                if name.lower() in words:
+                    if name not in found_competitors:
+                        found_competitors.append(name)
+                else:
+                    for alias in aliases:
+                        if alias.lower() in words:
+                            if name not in found_competitors:
+                                found_competitors.append(name)
+                            break
+            else:
+                if str(comp).lower() in words:
+                    found_competitors.append(str(comp))
+
+        return found_competitors
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a forum post through preprocessing pipeline.
+
+        Args:
+            input_data: Dictionary containing post data with at least:
+                - post_id: Post identifier
+                - post_content: Raw HTML content
+                - thread_title: Thread title (optional)
+                - thread_first_post: First post content (optional)
+                - category_title: Category title (optional)
+                - category_topic: Category topic (optional)
+
+        Returns:
+            Dictionary with preprocessing results
+        """
+        try:
+            # Validate input
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    **input_data
+                }
+
+            post_content = input_data.get("post_content", "")
+
+            # Step 1: Parse HTML content
+            parsed = self.html_parser.parse_post_content(post_content)
+            reply_content = parsed.get("reply_content", "")
+            quoted_content = parsed.get("quoted_content")
+
+            # Check for empty content
+            if not reply_content or len(reply_content.strip()) < 3:
+                return {
+                    "success": True,
+                    "cleaned_content": reply_content,
+                    "quoted_content": quoted_content,
+                    "is_empty": True,
+                    "preliminary_relevant": False,
+                    "needs_relevance_validation": False,
+                    **{k: v for k, v in input_data.items() if k != "post_content"}
+                }
+
+            # Step 2: Build thread context
+            thread_context = self.html_parser.build_thread_context(
+                thread_title=input_data.get("thread_title"),
+                first_post_content=input_data.get("thread_first_post"),
+                category_title=input_data.get("category_title"),
+                category_topic=input_data.get("category_topic")
+            )
+
+            # Step 3: Detect language
+            lang_result = self._detect_language(reply_content)
+
+            # Step 4: Check relevance - ONLY on the actual post content, NOT quoted/context
+            # The quoted content and thread context are for understanding, not for relevance determination
+            relevance_result = self._check_relevance(reply_content)
+
+            # Step 5: Extract product and competitor mentions - ONLY from actual post content
+            # We don't want to extract from quoted content as that will be processed separately
+            products_found = self._extract_mentioned_products(reply_content)
+            competitors_found = self._extract_mentioned_competitors(reply_content)
+
+            # Build result
+            result = {
+                "success": True,
+                "is_empty": False,
+
+                # Cleaned content
+                "cleaned_content": reply_content,
+                "quoted_content": quoted_content,
+                "has_quote": parsed.get("has_quote", False),
+                "quoted_author": parsed.get("quoted_author"),
+                "thread_context": thread_context,
+
+                # Language detection
+                "detected_language": lang_result["language"],
+                "language_code": lang_result["language_code"],
+                "is_english": lang_result["is_english"],
+                "language_confidence": lang_result["confidence"],
+
+                # Relevance assessment
+                "preliminary_relevant": relevance_result["preliminary_relevant"],
+                "needs_relevance_validation": relevance_result["needs_relevance_validation"],
+                "relevance_keywords_found": relevance_result["found_keywords"],
+                "relevance_type": relevance_result["relevance_type"],
+                "relevance_confidence": relevance_result["relevance_confidence"],
+
+                # Initial extractions
+                "products_detected": products_found,
+                "competitors_detected": competitors_found,
+
+                # Preserve original data
+                **{k: v for k, v in input_data.items() if k not in ["post_content"]}
+            }
+
+            # Keep original content for reference
+            result["original_content"] = post_content
+
+            self.log_processing(
+                f"Processed post {input_data.get('post_id')}: "
+                f"lang={lang_result['language']}, "
+                f"relevant={relevance_result['preliminary_relevant']}, "
+                f"needs_validation={relevance_result['needs_relevance_validation']}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, f"preprocessing post {input_data.get('post_id')}")

processing_brand_sentiment/workflow/agents/relevance_validator_agent.py

@@ -0,0 +1,289 @@
+"""
+Relevance Validator Agent for brand sentiment analysis.
+Lightweight LLM-based agent that confirms whether ambiguous terms (HH, AA)
+refer to Sabian products or generic terms.
+"""
+
+from typing import Dict, Any
+import json
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+import logging
+
+from .base_agent import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class RelevanceValidatorAgent(BaseAgent):
+    """
+    Agent that validates whether posts with ambiguous terms (like HH, AA)
+    are actually referring to Sabian products or generic terms.
+
+    This is a lightweight LLM call specifically for disambiguation.
+    """
+
+    def __init__(self, config: Dict[str, Any], api_key: str, brand_config: Dict[str, Any]):
+        """
+        Initialize the Relevance Validator Agent.
+
+        Args:
+            config: Agent configuration
+            api_key: OpenAI API key
+            brand_config: Brand-specific configuration with product info
+        """
+        super().__init__("RelevanceValidatorAgent", config)
+        self.api_key = api_key
+        self.brand_config = brand_config
+
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key
+        )
+
+        # Build disambiguation context from brand config
+        self._build_disambiguation_context()
+
+        logger.info("RelevanceValidatorAgent initialized")
+
+    def _build_disambiguation_context(self) -> None:
+        """Build context strings for disambiguation from brand config."""
+        brand = self.brand_config.get("brand", {})
+        ambiguous = brand.get("ambiguous_terms", {})
+
+        self.disambiguation_info = {}
+        for term, info in ambiguous.items():
+            if isinstance(info, dict):
+                self.disambiguation_info[term] = {
+                    "description": info.get("description", ""),
+                    "context_clues": info.get("disambiguation_context", [])
+                }
+            else:
+                self.disambiguation_info[term] = {
+                    "description": str(info),
+                    "context_clues": []
+                }
+
+        # Product descriptions for context
+        self.product_descriptions = brand.get("product_descriptions", {})
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate that input contains required fields.
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        required = ["cleaned_content", "relevance_keywords_found"]
+        return all(field in input_data for field in required)
+
+    def _build_system_prompt(self) -> str:
+        """Build the system prompt for relevance validation."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+        products = self.brand_config.get("brand", {}).get("products", [])
+
+        # Build disambiguation rules
+        disambiguation_rules = []
+        for term, info in self.disambiguation_info.items():
+            desc = info.get("description", "")
+            clues = info.get("context_clues", [])
+            rule = f"- '{term}': {desc}"
+            if clues:
+                rule += f" Context clues for {brand_name}: {', '.join(clues)}"
+            disambiguation_rules.append(rule)
+
+        disambiguation_text = "\n".join(disambiguation_rules) if disambiguation_rules else "No specific disambiguation rules."
+
+        system_prompt = f"""You are an expert at identifying brand mentions in drum/cymbal forum discussions.
+
+Your task is to determine if the POST CONTENT itself discusses {brand_name} products.
+
+**CRITICAL RULE:**
+- You must determine relevance based ONLY on the POST CONTENT
+- The context (thread info, quoted/parent content) is provided to help you understand ambiguous terms
+- But if the POST CONTENT itself does not mention or discuss {brand_name}, it is NOT relevant
+- Example: If quoted content mentions Sabian but the post just says "Got it! Thanks!" → NOT relevant
+
+**{brand_name} Product Lines:**
+{', '.join(products)}
+
+**Ambiguous Terms to Watch For:**
+{disambiguation_text}
+
+**Key Disambiguation Rules:**
+- "HH" alone usually means "Hi-Hat" (a type of cymbal), NOT Sabian HH series
+- "HH" WITH Sabian context IN THE POST (e.g., "Sabian HH", "HH crashes", "my HH ride") likely refers to Sabian
+- "AA" alone might be a general abbreviation, NOT Sabian AA series
+- "AA" WITH Sabian context IN THE POST (e.g., "Sabian AA", "AA cymbals", "AA medium ride") likely refers to Sabian
+- Generic replies like "Thanks!", "Got it!", "Good point!" are NOT relevant even if context mentions {brand_name}
+
+**Return JSON with:**
+- is_relevant: boolean - true ONLY if the POST CONTENT itself discusses {brand_name} products
+- confidence: "high", "medium", or "low"
+- reason: brief explanation (1-2 sentences) - explain what IN THE POST made you decide
+- detected_products: list of {brand_name} products mentioned IN THE POST (empty if none)
+
+Return only valid JSON."""
+
+        return system_prompt
+
+    def validate_relevance(
+        self,
+        content: str,
+        keywords_found: list,
+        thread_context: str = "",
+        quoted_content: str = ""
+    ) -> Dict[str, Any]:
+        """
+        Validate whether content is relevant to the brand.
+
+        Args:
+            content: The cleaned post content
+            keywords_found: Keywords that triggered validation
+            thread_context: Thread context for additional context
+            quoted_content: Quoted content if any
+
+        Returns:
+            Dictionary with validation results
+        """
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+
+        # Build context for the LLM
+        context_parts = []
+        if thread_context:
+            context_parts.append(f"Thread context: {thread_context}")
+        if quoted_content:
+            context_parts.append(f"Replying to: {quoted_content[:300]}...")
+
+        context_str = "\n".join(context_parts) if context_parts else "No additional context."
+
+        user_prompt = f"""Determine if this POST CONTENT discusses {brand_name} cymbal products.
+
+**Keywords found in post:** {', '.join(keywords_found)}
+
+**CONTEXT (for understanding ambiguous terms only - do NOT base relevance on this):**
+{context_str}
+
+**POST CONTENT TO EVALUATE (base your relevance decision ONLY on this):**
+"{content}"
+
+Does the POST CONTENT itself discuss {brand_name} products? Remember: generic replies are NOT relevant even if context mentions {brand_name}. Return JSON only."""
+
+        try:
+            messages = [
+                SystemMessage(content=self._build_system_prompt()),
+                HumanMessage(content=user_prompt)
+            ]
+
+            response = self.llm.invoke(messages)
+            result = self._parse_llm_json_response(response.content)
+
+            return {
+                "success": True,
+                "is_relevant": result.get("is_relevant", False),
+                "relevance_confidence": result.get("confidence", "low"),
+                "relevance_reason": result.get("reason", ""),
+                "detected_products": result.get("detected_products", [])
+            }
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"JSON decode error in relevance validation: {e}", "warning")
+            # Default to relevant if we can't determine
+            return {
+                "success": True,
+                "is_relevant": True,
+                "relevance_confidence": "low",
+                "relevance_reason": "Could not parse LLM response, defaulting to relevant",
+                "detected_products": []
+            }
+
+        except Exception as e:
+            self.log_processing(f"Relevance validation error: {e}", "error")
+            return {
+                "success": False,
+                "is_relevant": True,  # Default to relevant on error
+                "relevance_confidence": "low",
+                "relevance_reason": f"Error during validation: {str(e)}",
+                "detected_products": [],
+                "error": str(e)
+            }
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a post to validate its relevance to the brand.
+
+        Args:
+            input_data: Dictionary containing:
+                - cleaned_content: Cleaned post text
+                - relevance_keywords_found: Keywords that triggered validation
+                - thread_context: Optional thread context
+                - quoted_content: Optional quoted content
+
+        Returns:
+            Dictionary with validation results and original data
+        """
+        try:
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    "is_relevant": True,  # Default to relevant
+                    "relevance_confidence": "low",
+                    **input_data
+                }
+
+            # Check if validation is actually needed
+            if not input_data.get("needs_relevance_validation", False):
+                # No validation needed, use preliminary assessment
+                return {
+                    "success": True,
+                    "is_relevant": input_data.get("preliminary_relevant", False),
+                    "relevance_confidence": input_data.get("relevance_confidence", "high"),
+                    "relevance_reason": "No validation needed - preliminary assessment used",
+                    "validation_performed": False,
+                    **input_data
+                }
+
+            # Perform LLM validation
+            validation_result = self.validate_relevance(
+                content=input_data.get("cleaned_content", ""),
+                keywords_found=input_data.get("relevance_keywords_found", []),
+                thread_context=input_data.get("thread_context", ""),
+                quoted_content=input_data.get("quoted_content", "")
+            )
+
+            # Merge results
+            result = {
+                **input_data,
+                "is_relevant": validation_result["is_relevant"],
+                "relevance_confidence": validation_result["relevance_confidence"],
+                "relevance_reason": validation_result["relevance_reason"],
+                "validation_performed": True,
+                "success": validation_result["success"]
+            }
+
+            # Update products detected if LLM found any
+            if validation_result.get("detected_products"):
+                existing_products = input_data.get("products_detected", [])
+                llm_products = validation_result["detected_products"]
+                # Merge without duplicates
+                all_products = list(set(existing_products + llm_products))
+                result["products_detected"] = all_products
+
+            if "error" in validation_result:
+                result["validation_error"] = validation_result["error"]
+
+            self.log_processing(
+                f"Validated relevance for post: is_relevant={result['is_relevant']}, "
+                f"confidence={result['relevance_confidence']}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "relevance validation")

processing_brand_sentiment/workflow/agents/sabian_analyzer_agent.py

@@ -0,0 +1,388 @@
+"""
+Sabian Analyzer Agent for comprehensive brand sentiment analysis.
+LLM-based agent that extracts products, competitors, sentiment, intents,
+pain points, and other brand intelligence from forum posts.
+"""
+
+from typing import Dict, Any, List
+import json
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+import logging
+
+from .base_agent import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class SabianAnalyzerAgent(BaseAgent):
+    """
+    Comprehensive brand analysis agent for Sabian cymbal discussions.
+    """
+
+    def __init__(
+        self,
+        config: Dict[str, Any],
+        api_key: str,
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any]
+    ):
+        super().__init__("SabianAnalyzerAgent", config)
+        self.api_key = api_key
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key
+        )
+
+        # Pre-compute valid values for validation
+        self._valid_values = self._compute_valid_values()
+        logger.info("SabianAnalyzerAgent initialized")
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        required = ["cleaned_content", "is_relevant"]
+        return all(field in input_data for field in required)
+
+    def _compute_valid_values(self) -> Dict[str, List[str]]:
+        """Pre-compute all valid values from config for validation."""
+        valid = {}
+
+        # Products from brand config
+        valid["products"] = self.brand_config.get("brand", {}).get("products", [])
+
+        # Competitors
+        competitor_names = []
+        for comp in self.brand_config.get("brand", {}).get("competitors", []):
+            if isinstance(comp, dict):
+                competitor_names.append(comp.get("name", ""))
+        valid["competitors"] = competitor_names
+
+        # Extract category values from analysis_categories
+        category_map = {
+            "author_role": "author_role",
+            "sabian_mention_context": "sabian_mention_context",
+            "sentiment_level": "sentiment",
+            "emotion_type": "emotions",
+            "intents": "intents",
+            "purchase_stage": "purchase_stage",
+            "comparison_type": "comparison_type",
+            "feedback_aspects": "feedback_aspects",
+            "decision_drivers": "decision_drivers",
+            "product_attributes": "product_attributes",
+        }
+
+        for key, config_key in category_map.items():
+            config_section = self.analysis_categories.get(config_key, {})
+            if "categories" in config_section:
+                valid[key] = [c["value"] for c in config_section["categories"]]
+            elif "levels" in config_section:
+                valid[key] = [c["value"] for c in config_section["levels"]]
+            else:
+                valid[key] = []
+
+        return valid
+
+    def _get_category_list(self, key: str) -> List[str]:
+        """Get list of valid values for a category."""
+        return self._valid_values.get(key, [])
+
+    def _build_system_prompt(self) -> str:
+        """Build optimized system prompt for brand analysis."""
+        brand = self.brand_config.get("brand", {})
+        brand_name = brand.get("name", "Sabian")
+        products = brand.get("products", [])
+
+        competitors = [c.get("name", "") for c in brand.get("competitors", []) if isinstance(c, dict)]
+
+        # Get all valid values
+        v = self._valid_values
+
+        return f"""You are a brand analyst extracting insights from forum posts about {brand_name} cymbals.
+
+## STRICT RULES
+1. Extract ONLY from POST CONTENT, never from quoted/context text
+2. Use ONLY values from the lists below - return null/[] if no match
+3. Sentiment must be about {brand_name} specifically, NOT overall post tone
+4. pain_points/delight_factors use SAME value list (feedback_aspects) - classification determines positive vs negative
+
+## VALID VALUES
+
+**{brand_name} Products:** {products}
+**Competitors:** {competitors}
+
+| Field | Valid Values |
+|-------|--------------|
+| author_role | {v.get('author_role', [])} |
+| sabian_mention_context | {v.get('sabian_mention_context', [])} |
+| sentiment_level | {v.get('sentiment_level', [])} |
+| emotion_type | {v.get('emotion_type', [])} |
+| intents (multi) | {v.get('intents', [])} |
+| purchase_stage | {v.get('purchase_stage', [])} |
+| comparison_type | {v.get('comparison_type', [])} |
+| feedback_aspects | {v.get('feedback_aspects', [])} |
+| decision_drivers | {v.get('decision_drivers', [])} |
+| product_attributes | {v.get('product_attributes', [])} |
+
+## KEY DISTINCTIONS
+
+**Sentiment vs Intent:**
+- sentiment_level = How author FEELS about {brand_name} (positive/negative/neutral)
+- praising/criticizing intent = Author is actively ENDORSING or WARNING others
+
+**Author-only fields (null if giving advice to others):**
+- purchase_stage, decision_drivers, pain_points, delight_factors
+
+**Example - Sabian-specific sentiment:**
+Post: "Love my new drum kit! The SBR cymbals sound terrible though."
+- Overall post: positive (happy about kit)
+- {brand_name} sentiment: NEGATIVE (dislikes SBR sound)
+- pain_points: ["sound_quality"]
+
+## OUTPUT JSON
+```json
+{{
+  "author_role": "value from list",
+  "sabian_mention_context": "value from list",
+  "sentiment_level": "value from list",
+  "emotion_type": "value or null",
+  "sentiment_confidence": "high|medium|low",
+  "sarcasm_detected": false,
+  "products_mentioned": [],
+  "product_attributes": [],
+  "competitors_mentioned": [],
+  "competitor_products_owned": [],
+  "comparison_type": "value or null",
+  "intents": [],
+  "purchase_stage": "value or null",
+  "decision_drivers": [],
+  "pain_points": [],
+  "delight_factors": [],
+  "analysis_notes": "1-2 sentences on key {brand_name}-specific insights"
+}}
+```
+
+Return ONLY valid JSON."""
+
+    def analyze_post(
+        self,
+        content: str,
+        thread_context: str = "",
+        quoted_content: str = ""
+    ) -> Dict[str, Any]:
+        """Perform brand analysis on a post."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+
+        context_str = ""
+        if thread_context:
+            context_str += f"[Thread: {thread_context[:200]}] "
+        if quoted_content:
+            context_str += f"[Replying to: {quoted_content[:200]}...]"
+
+        user_prompt = f"""Analyze this post about {brand_name}.
+
+CONTEXT (for understanding only, DO NOT extract from): {context_str or "None"}
+
+POST CONTENT (extract from THIS only):
+"{content}"
+
+Return JSON only."""
+
+        try:
+            messages = [
+                SystemMessage(content=self._build_system_prompt()),
+                HumanMessage(content=user_prompt)
+            ]
+
+            response = self.llm.invoke(messages)
+            result = self._parse_llm_json_response(response.content)
+            validated = self._validate_and_normalize(result)
+
+            return {"success": True, **validated}
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"JSON decode error: {e}", "warning")
+            return {
+                "success": False,
+                "error": f"JSON parse error: {str(e)}",
+                "sentiment_level": "neutral",
+                "intents": ["general_discussion"]
+            }
+        except Exception as e:
+            self.log_processing(f"Analysis error: {e}", "error")
+            return {"success": False, "error": str(e)}
+
+    def _validate_single(self, value: Any, valid_list: List[str], default: Any = None) -> Any:
+        """Validate single value against list, return canonical form or default."""
+        if value is None:
+            return default
+        if isinstance(value, str):
+            val_lower = value.lower()
+            for v in valid_list:
+                if v.lower() == val_lower:
+                    return v
+        return default
+
+    def _validate_list(self, values: Any, valid_list: List[str]) -> List[str]:
+        """Validate list values, return only valid items in canonical form."""
+        if not values:
+            return []
+        if not isinstance(values, list):
+            values = [values]
+
+        validated = []
+        valid_lower = {v.lower(): v for v in valid_list}
+        for val in values:
+            if isinstance(val, str) and val.lower() in valid_lower:
+                validated.append(valid_lower[val.lower()])
+        return validated
+
+    def _validate_and_normalize(self, result: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate all fields against predefined values and normalize."""
+        v = self._valid_values
+
+        normalized = {
+            # Classification
+            "author_role": self._validate_single(
+                result.get("author_role"), v["author_role"], "unknown"
+            ),
+            "sabian_mention_context": self._validate_single(
+                result.get("sabian_mention_context"), v["sabian_mention_context"], "casual_mention"
+            ),
+
+            # Sentiment
+            "sentiment_level": self._validate_single(
+                result.get("sentiment_level"), v["sentiment_level"], "neutral"
+            ),
+            "emotion_type": self._validate_single(
+                result.get("emotion_type"), v["emotion_type"], None
+            ),
+            "sentiment_confidence": result.get("sentiment_confidence", "medium"),
+            "sarcasm_detected": bool(result.get("sarcasm_detected", False)),
+
+            # Products
+            "products_mentioned": self._validate_list(
+                result.get("products_mentioned"), v["products"]
+            ),
+            "product_attributes": self._validate_list(
+                result.get("product_attributes"), v["product_attributes"]
+            ),
+
+            # Competitors
+            "competitors_mentioned": self._validate_list(
+                result.get("competitors_mentioned"), v["competitors"]
+            ),
+            "competitor_products_owned": self._validate_list(
+                result.get("competitor_products_owned"), v["competitors"]
+            ),
+            "comparison_type": self._validate_single(
+                result.get("comparison_type"), v["comparison_type"], None
+            ),
+
+            # Intents
+            "intents": self._validate_list(
+                result.get("intents"), v["intents"]
+            ) or ["general_discussion"],
+
+            # Author journey (null if advising others)
+            "purchase_stage": self._validate_single(
+                result.get("purchase_stage"), v["purchase_stage"], None
+            ),
+            "decision_drivers": self._validate_list(
+                result.get("decision_drivers"), v["decision_drivers"]
+            ),
+
+            # Feedback - both use feedback_aspects
+            "pain_points": self._validate_list(
+                result.get("pain_points"), v["feedback_aspects"]
+            ),
+            "delight_factors": self._validate_list(
+                result.get("delight_factors"), v["feedback_aspects"]
+            ),
+
+            # Notes
+            "analysis_notes": result.get("analysis_notes", ""),
+        }
+
+        # Log filtered values for debugging
+        for field in ["products_mentioned", "product_attributes", "pain_points", "delight_factors"]:
+            original = result.get(field, [])
+            if isinstance(original, list) and len(original) > len(normalized[field]):
+                filtered = set(str(x) for x in original) - set(normalized[field])
+                if filtered:
+                    logger.debug(f"Filtered invalid {field}: {filtered}")
+
+        return normalized
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Process a post through brand analysis."""
+        try:
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    **input_data
+                }
+
+            # Skip non-relevant posts
+            if not input_data.get("is_relevant", False):
+                return {
+                    "success": True,
+                    "analysis_skipped": True,
+                    "analysis_skip_reason": "Post marked as not relevant",
+                    "author_role": None,
+                    "sabian_mention_context": None,
+                    "sentiment_level": None,
+                    "emotion_type": None,
+                    "products_mentioned": [],
+                    "competitors_mentioned": [],
+                    "competitor_products_owned": [],
+                    "intents": [],
+                    "purchase_stage": None,
+                    "decision_drivers": [],
+                    "pain_points": [],
+                    "delight_factors": [],
+                    **input_data
+                }
+
+            # Skip non-English posts
+            if not input_data.get("is_english", True):
+                return {
+                    "success": True,
+                    "analysis_skipped": True,
+                    "analysis_skip_reason": f"Non-English: {input_data.get('detected_language')}",
+                    "author_role": None,
+                    "sabian_mention_context": None,
+                    "sentiment_level": None,
+                    "emotion_type": None,
+                    "intents": [],
+                    "competitor_products_owned": [],
+                    **input_data
+                }
+
+            # Perform analysis
+            analysis_result = self.analyze_post(
+                content=input_data.get("cleaned_content", ""),
+                thread_context=input_data.get("thread_context", ""),
+                quoted_content=input_data.get("quoted_content", "")
+            )
+
+            result = {
+                **input_data,
+                **analysis_result,
+                "analysis_skipped": False
+            }
+
+            self.log_processing(
+                f"Analyzed: sentiment={result.get('sentiment_level')}, "
+                f"products={len(result.get('products_mentioned', []))}, "
+                f"intents={result.get('intents', [])}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "brand analysis")

processing_brand_sentiment/workflow/agents/sabian_relevance_extraction_agent.py

@@ -0,0 +1,431 @@
+"""
+Sabian Relevance & Extraction Agent for brand sentiment analysis.
+
+This agent performs two critical functions:
+1. Determines relevance with HIGH confidence using strict rules
+2. Extracts verifiable facts (products, author role, context summary)
+
+Key Design Principles:
+- Strict product matching: ONLY return products from predefined list
+- Competitor awareness: Know what products belong to competitors
+- Conservative relevance: When uncertain, mark as NOT relevant
+- Thread context summarization: Provide clean, concise context for next agent
+"""
+
+from typing import Dict, Any, List
+import json
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+import logging
+
+from .base_agent import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class SabianRelevanceExtractionAgent(BaseAgent):
+    """
+    Agent that validates relevance and extracts key facts from posts.
+
+    This agent is the first LLM call in the pipeline and serves as the
+    gatekeeper for relevance while also extracting structured information
+    for downstream analysis.
+    """
+
+    def __init__(
+        self,
+        config: Dict[str, Any],
+        api_key: str,
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any]
+    ):
+        """
+        Initialize the Relevance & Extraction Agent.
+
+        Args:
+            config: Agent configuration
+            api_key: OpenAI API key
+            brand_config: Brand-specific configuration with products and competitors
+            analysis_categories: Category definitions for validation
+        """
+        super().__init__("SabianRelevanceExtractionAgent", config)
+        self.api_key = api_key
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key
+        )
+
+        # Pre-compute valid values
+        self._build_valid_values()
+        self._build_competitor_product_warnings()
+
+        logger.info("SabianRelevanceExtractionAgent initialized")
+
+    def _build_valid_values(self) -> None:
+        """Build valid value lists for validation."""
+        brand = self.brand_config.get("brand", {})
+
+        # Products
+        self.valid_products = brand.get("products", [])
+
+        # Competitors (brand names only)
+        self.valid_competitors = []
+        for comp in brand.get("competitors", []):
+            if isinstance(comp, dict):
+                self.valid_competitors.append(comp.get("name", ""))
+            else:
+                self.valid_competitors.append(str(comp))
+
+        # Author roles from categories
+        author_role_config = self.analysis_categories.get("author_role", {})
+        self.valid_author_roles = [
+            c["value"] for c in author_role_config.get("categories", [])
+        ]
+
+        # Sabian mention context from categories
+        mention_context_config = self.analysis_categories.get("sabian_mention_context", {})
+        self.valid_mention_contexts = [
+            c["value"] for c in mention_context_config.get("categories", [])
+        ]
+
+    def _build_competitor_product_warnings(self) -> None:
+        """Build list of competitor products to warn about in prompts."""
+        warnings = self.brand_config.get("brand", {}).get("competitor_products_warning", {})
+
+        self.competitor_products_by_brand = {}
+        for key, products in warnings.items():
+            if key == "description":
+                continue
+            # Extract brand name from key (e.g., "paiste_products" -> "Paiste")
+            brand_name = key.replace("_products", "").capitalize()
+            self.competitor_products_by_brand[brand_name] = products
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """Validate input contains required fields."""
+        required = ["cleaned_content"]
+        return all(field in input_data for field in required)
+
+    def _build_system_prompt(self) -> str:
+        """Build the system prompt for relevance and extraction."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+
+        # Build competitor product warnings
+        competitor_warnings = []
+        for brand, products in self.competitor_products_by_brand.items():
+            products_str = ", ".join(f'"{p}"' for p in products[:5])  # Limit to 5 examples
+            if len(products) > 5:
+                products_str += f" (and {len(products)-5} more)"
+            competitor_warnings.append(f"- {brand}: {products_str}")
+
+        competitor_warnings_text = "\n".join(competitor_warnings) if competitor_warnings else "None specified"
+
+        return f"""You are a brand mention extractor for {brand_name} cymbals. Your job is to:
+1. Determine if the POST CONTENT discusses {brand_name} products or brand
+2. Extract ONLY verifiable facts, not interpretations
+
+## CRITICAL RULES
+
+### Rule 1: Relevance Based on POST CONTENT Only
+- The post is relevant ONLY if the POST CONTENT itself mentions {brand_name} brand or products
+- Quoted/parent content mentioning {brand_name} does NOT make the post relevant
+- Generic replies ("Thanks!", "Got it!", "Good point!") are NEVER relevant
+- Posts can be relevant even without specific product mentions if they discuss the {brand_name} brand
+
+### Rule 2: Strict Product Matching
+{brand_name.upper()} PRODUCTS (use ONLY these exact values):
+{self.valid_products}
+
+CRITICAL:
+- Return ONLY products from this exact list above
+- If you see a product not in this list, do NOT include it
+- Return empty list [] if no products from the list are mentioned
+- It's OK to have empty products_mentioned if the post discusses {brand_name} brand generally
+
+### Rule 3: Competitor Product Awareness
+These products belong to COMPETITORS, NOT {brand_name}:
+{competitor_warnings_text}
+
+COMPETITOR BRANDS: {self.valid_competitors}
+- Only return competitor BRAND names in competitors_mentioned (not their products)
+- If you see "2002", "Signature", "Sound Edge", "Formula 602" - these are PAISTE, not {brand_name}
+- If you see "K Custom", "A Custom" - these are ZILDJIAN, not {brand_name}
+
+### Rule 4: Thread Context Summary
+- Summarize thread context in 1-2 sentences MAXIMUM
+- Focus only on what helps understand what the post is responding to
+- If thread is about unrelated topics (pizza, general life), say so briefly
+- Keep it factual and concise
+
+### Rule 5: Author Role Classification
+Determine the author's relationship to {brand_name}:
+- current_owner: Currently owns/uses {brand_name} products
+- past_owner: Previously owned but sold/replaced
+- potential_buyer: Considering purchasing {brand_name}
+- never_owned: Explicitly states they don't own {brand_name}
+- unknown: Cannot determine from post content
+
+### Rule 6: Mention Context Classification
+How prominently is {brand_name} discussed IN THE POST CONTENT:
+- primary_focus: {brand_name} is the main topic of the post
+- significant_mention: {brand_name} discussed with some detail, but not main focus
+- casual_mention: Brief mention among other topics
+- comparison_context: Mentioned while comparing to competitors
+- null: Not relevant (use when is_relevant=false)
+
+## OUTPUT FORMAT
+Return ONLY valid JSON with these exact fields:
+```json
+{{
+    "is_relevant": true/false,
+    "relevance_confidence": "high" | "medium" | "low",
+    "relevance_reason": "1-2 sentences explaining your decision",
+    "products_mentioned": [],
+    "sabian_mention_context": "value from list" | null,
+    "author_role": "value from list",
+    "competitors_mentioned": [],
+    "thread_context_summary": "1-2 sentence summary of thread context"
+}}
+```
+
+IMPORTANT: Return ONLY the JSON object, no additional text."""
+
+    def _build_user_prompt(
+        self,
+        content: str,
+        quoted_content: str,
+        raw_thread_context: str,
+        keywords_found: List[str]
+    ) -> str:
+        """Build the user prompt with post content and context."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+
+        context_section = ""
+        if raw_thread_context:
+            # Truncate if too long
+            truncated_context = raw_thread_context[:1000] if len(raw_thread_context) > 1000 else raw_thread_context
+            context_section += f"THREAD CONTEXT (for understanding only):\n{truncated_context}\n\n"
+
+        if quoted_content:
+            truncated_quote = quoted_content[:500] if len(quoted_content) > 500 else quoted_content
+            context_section += f"QUOTED/PARENT CONTENT (for understanding only):\n{truncated_quote}\n\n"
+
+        keywords_info = ""
+        if keywords_found:
+            keywords_info = f"Keywords detected by preprocessor: {', '.join(keywords_found)}\n\n"
+
+        return f"""Analyze this post for {brand_name} relevance and extract facts.
+
+{keywords_info}{context_section}POST CONTENT TO EVALUATE (base your decision ONLY on this):
+\"\"\"{content}\"\"\"
+
+Remember:
+- is_relevant=true ONLY if POST CONTENT discusses {brand_name}
+- products_mentioned must be from the exact product list provided
+- competitors_mentioned should be brand names only (Zildjian, Paiste, etc.)
+- thread_context_summary should be 1-2 sentences max
+
+Return JSON only."""
+
+    def extract_and_validate(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Perform relevance check and fact extraction.
+
+        Args:
+            input_data: Preprocessed post data
+
+        Returns:
+            Dictionary with extraction results
+        """
+        content = input_data.get("cleaned_content", "")
+        quoted_content = input_data.get("quoted_content", "")
+        raw_thread_context = input_data.get("raw_thread_context", "")
+        keywords_found = input_data.get("relevance_keywords_found", [])
+
+        try:
+            messages = [
+                SystemMessage(content=self._build_system_prompt()),
+                HumanMessage(content=self._build_user_prompt(
+                    content, quoted_content, raw_thread_context, keywords_found
+                ))
+            ]
+
+            response = self.llm.invoke(messages)
+            result = self._parse_llm_json_response(response.content)
+
+            # Validate and normalize the response
+            validated = self._validate_response(result)
+
+            return {
+                "success": True,
+                **validated
+            }
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"JSON decode error: {e}", "warning")
+            return {
+                "success": False,
+                "error": f"JSON parse error: {str(e)}",
+                "is_relevant": False,
+                "relevance_confidence": "low",
+                "relevance_reason": "Failed to parse LLM response"
+            }
+
+        except Exception as e:
+            self.log_processing(f"Extraction error: {e}", "error")
+            return {
+                "success": False,
+                "error": str(e),
+                "is_relevant": False,
+                "relevance_confidence": "low",
+                "relevance_reason": f"Error during extraction: {str(e)}"
+            }
+
+    def _validate_response(self, result: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate and normalize LLM response against allowed values."""
+
+        # Validate products
+        products = result.get("products_mentioned", [])
+        if not isinstance(products, list):
+            products = []
+        valid_products = [
+            p for p in products
+            if any(p.lower() == vp.lower() for vp in self.valid_products)
+        ]
+        # Normalize to canonical case
+        normalized_products = []
+        for p in valid_products:
+            for vp in self.valid_products:
+                if p.lower() == vp.lower():
+                    normalized_products.append(vp)
+                    break
+
+        # Validate competitors
+        competitors = result.get("competitors_mentioned", [])
+        if not isinstance(competitors, list):
+            competitors = []
+        valid_competitors = [
+            c for c in competitors
+            if any(c.lower() == vc.lower() for vc in self.valid_competitors)
+        ]
+        # Normalize to canonical case
+        normalized_competitors = []
+        for c in valid_competitors:
+            for vc in self.valid_competitors:
+                if c.lower() == vc.lower():
+                    normalized_competitors.append(vc)
+                    break
+
+        # Validate author_role
+        author_role = result.get("author_role", "unknown")
+        if author_role not in self.valid_author_roles:
+            author_role = "unknown"
+
+        # Validate sabian_mention_context
+        mention_context = result.get("sabian_mention_context")
+        is_relevant = result.get("is_relevant", False)
+
+        if not is_relevant:
+            mention_context = None
+        elif mention_context and mention_context not in self.valid_mention_contexts:
+            mention_context = "casual_mention"  # Default for relevant posts
+
+        # Validate confidence
+        confidence = result.get("relevance_confidence", "medium")
+        if confidence not in ["high", "medium", "low"]:
+            confidence = "medium"
+
+        return {
+            "is_relevant": bool(is_relevant),
+            "relevance_confidence": confidence,
+            "relevance_reason": result.get("relevance_reason", ""),
+            "products_mentioned": normalized_products,
+            "sabian_mention_context": mention_context,
+            "author_role": author_role,
+            "competitors_mentioned": normalized_competitors,
+            "thread_context_summary": result.get("thread_context_summary", "")
+        }
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a post through relevance validation and fact extraction.
+
+        Args:
+            input_data: Dictionary from preprocessor containing:
+                - cleaned_content: Cleaned post text
+                - quoted_content: Quoted content if any
+                - raw_thread_context: Raw thread context
+                - relevance_keywords_found: Keywords from preprocessor
+                - preliminary_relevant: Preprocessor's relevance assessment
+                - needs_relevance_validation: Whether LLM validation needed
+
+        Returns:
+            Dictionary with extraction results and original data
+        """
+        try:
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    "is_relevant": False,
+                    **input_data
+                }
+
+            # Skip if already determined not relevant and no validation needed
+            if (not input_data.get("preliminary_relevant", False) and
+                not input_data.get("needs_relevance_validation", False)):
+                return {
+                    "success": True,
+                    "is_relevant": False,
+                    "relevance_confidence": "high",
+                    "relevance_reason": "No Sabian-related keywords found in post",
+                    "products_mentioned": [],
+                    "sabian_mention_context": None,
+                    "author_role": "unknown",
+                    "competitors_mentioned": input_data.get("competitors_detected", []),
+                    "thread_context_summary": "",
+                    "extraction_performed": False,
+                    **input_data
+                }
+
+            # Skip non-English posts
+            if not input_data.get("is_english", True):
+                return {
+                    "success": True,
+                    "is_relevant": False,
+                    "relevance_confidence": "high",
+                    "relevance_reason": f"Non-English post: {input_data.get('detected_language')}",
+                    "products_mentioned": [],
+                    "sabian_mention_context": None,
+                    "author_role": "unknown",
+                    "competitors_mentioned": [],
+                    "thread_context_summary": "",
+                    "extraction_performed": False,
+                    **input_data
+                }
+
+            # Perform LLM extraction
+            extraction_result = self.extract_and_validate(input_data)
+
+            # Merge results
+            result = {
+                **input_data,
+                **extraction_result,
+                "extraction_performed": True
+            }
+
+            # Log the result
+            self.log_processing(
+                f"Extraction complete: is_relevant={result.get('is_relevant')}, "
+                f"products={result.get('products_mentioned')}, "
+                f"context={result.get('sabian_mention_context')}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "relevance extraction")

processing_brand_sentiment/workflow/agents/sabian_sentiment_analyzer_agent.py

@@ -0,0 +1,434 @@
+"""
+Sabian Sentiment & Intent Analyzer Agent for brand sentiment analysis.
+
+This agent performs deep analysis on VERIFIED relevant posts with STRUCTURED input.
+It receives pre-validated data from the Relevance Extraction Agent including:
+- Products already extracted and validated
+- Thread context already summarized
+- Author role already determined
+
+Key Design Principles:
+- Focused analysis: Only sentiment, intents, and customer journey
+- No re-extraction: Products are given, not re-detected
+- Sabian-specific sentiment: How author feels about Sabian, not overall post tone
+- Author perspective: Pain points/delights only from author's own experience
+"""
+
+from typing import Dict, Any, List
+import json
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+import logging
+
+from .base_agent import BaseAgent
+
+logger = logging.getLogger(__name__)
+
+
+class SabianSentimentAnalyzerAgent(BaseAgent):
+    """
+    Agent that performs deep sentiment and intent analysis on relevant posts.
+
+    This agent is the second LLM call in the pipeline and focuses purely on
+    analysis, not extraction. It receives structured input from the extraction
+    agent and produces sentiment, intent, and customer journey insights.
+    """
+
+    def __init__(
+        self,
+        config: Dict[str, Any],
+        api_key: str,
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any]
+    ):
+        """
+        Initialize the Sentiment Analyzer Agent.
+
+        Args:
+            config: Agent configuration
+            api_key: OpenAI API key
+            brand_config: Brand-specific configuration
+            analysis_categories: Category definitions for analysis
+        """
+        super().__init__("SabianSentimentAnalyzerAgent", config)
+        self.api_key = api_key
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key
+        )
+
+        # Pre-compute valid values for validation
+        self._valid_values = self._compute_valid_values()
+
+        logger.info("SabianSentimentAnalyzerAgent initialized")
+
+    def _compute_valid_values(self) -> Dict[str, List[str]]:
+        """Pre-compute all valid values from config for validation."""
+        valid = {}
+
+        # Products from brand config
+        valid["products"] = self.brand_config.get("brand", {}).get("products", [])
+
+        # Competitors
+        competitor_names = []
+        for comp in self.brand_config.get("brand", {}).get("competitors", []):
+            if isinstance(comp, dict):
+                competitor_names.append(comp.get("name", ""))
+        valid["competitors"] = competitor_names
+
+        # Extract category values from analysis_categories
+        category_map = {
+            "sentiment_level": "sentiment",
+            "emotion_type": "emotions",
+            "intents": "intents",
+            "purchase_stage": "purchase_stage",
+            "comparison_type": "comparison_type",
+            "feedback_aspects": "feedback_aspects",
+            "decision_drivers": "decision_drivers",
+            "product_attributes": "product_attributes",
+        }
+
+        for key, config_key in category_map.items():
+            config_section = self.analysis_categories.get(config_key, {})
+            if "categories" in config_section:
+                valid[key] = [c["value"] for c in config_section["categories"]]
+            elif "levels" in config_section:
+                valid[key] = [c["value"] for c in config_section["levels"]]
+            else:
+                valid[key] = []
+
+        return valid
+
+    def _get_valid_list(self, key: str) -> List[str]:
+        """Get list of valid values for a category."""
+        return self._valid_values.get(key, [])
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """Validate that input contains required fields."""
+        required = ["cleaned_content", "is_relevant"]
+        return all(field in input_data for field in required)
+
+    def _build_system_prompt(self) -> str:
+        """Build optimized system prompt for sentiment analysis."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+        v = self._valid_values
+
+        return f"""You are a sentiment analyst for {brand_name} cymbal discussions.
+
+## YOUR TASK
+Analyze the sentiment, emotions, and intents in posts about {brand_name}.
+You will receive PRE-VALIDATED context (products, author role, etc.) - trust these values.
+
+## CRITICAL RULES
+
+### Rule 1: Neutral by Default
+Sentiment defaults to NEUTRAL unless there is EXPLICIT positive or negative language toward {brand_name}.
+- Factual statements = neutral
+- Comparative statements ("sounds different", "not the same as") = neutral (different ≠ worse)
+- Advice-giving without personal opinion = neutral
+
+Only assign positive/negative sentiment when the author CLEARLY expresses satisfaction or dissatisfaction with {brand_name}.
+
+### Rule 2: {brand_name}-Specific Sentiment
+Sentiment MUST be about {brand_name} specifically, NOT overall post tone or other products.
+
+EXAMPLE:
+Post: "I have SBR cymbals and bought a Pearl crash. The Pearl sounds different from the SBR. Go with what feels best!"
+- This is NEUTRAL toward {brand_name} - "different" is not criticism
+- The author owns SBR (no complaint), is giving advice
+- pain_points: [] (no negative experience expressed)
+- delight_factors: [] (no positive experience expressed)
+
+### Rule 3: Mutually Exclusive Feedback
+pain_points and delight_factors CANNOT contain the same values.
+- If an aspect is positive → delight_factors only
+- If an aspect is negative → pain_points only
+- Never both
+
+### Rule 4: Author Perspective Only
+These fields are ONLY for author's OWN experience, not advice to others:
+- purchase_stage, decision_drivers, pain_points, delight_factors
+
+If author is primarily giving ADVICE to someone else, these should be null/empty.
+
+### Rule 5: Valid Values
+
+| Field | Valid Values |
+|-------|--------------|
+| sentiment_level | {v.get('sentiment_level', [])} |
+| emotion_type | {v.get('emotion_type', [])} |
+| intents (multi-select) | {v.get('intents', [])} |
+| purchase_stage | {v.get('purchase_stage', [])} |
+| comparison_type | {v.get('comparison_type', [])} |
+| feedback_aspects | {v.get('feedback_aspects', [])} |
+| decision_drivers | {v.get('decision_drivers', [])} |
+| product_attributes | {v.get('product_attributes', [])} |
+| competitor brands | {v.get('competitors', [])} |
+
+### Rule 6: Intent Classification
+- seeking_information: Asking questions, seeking advice
+- providing_information: Answering questions, giving advice
+- sharing_experience: Personal experience, review, testimonial
+- comparing: Comparing brands/products
+- praising: Actively endorsing {brand_name}
+- criticizing: Actively complaining about {brand_name}
+- buying_selling: Listing gear for sale/trade
+- general_discussion: General conversation
+
+## OUTPUT FORMAT
+```json
+{{
+    "sentiment_level": "neutral unless explicit positive/negative",
+    "emotion_type": "value or null",
+    "sentiment_confidence": "high" | "medium" | "low",
+    "sarcasm_detected": false,
+    "product_attributes": [],
+    "competitor_products_owned": [],
+    "comparison_type": "value or null",
+    "intents": [],
+    "purchase_stage": "value or null",
+    "decision_drivers": [],
+    "pain_points": [],
+    "delight_factors": [],
+    "analysis_notes": "1-2 sentences"
+}}
+```
+
+Return ONLY valid JSON."""
+
+    def _build_user_prompt(self, input_data: Dict[str, Any]) -> str:
+        """Build user prompt with structured context."""
+        brand_name = self.brand_config.get("brand", {}).get("name", "Sabian")
+
+        content = input_data.get("cleaned_content", "")
+        products_mentioned = input_data.get("products_mentioned", [])
+        sabian_context = input_data.get("sabian_mention_context", "")
+        author_role = input_data.get("author_role", "unknown")
+        thread_summary = input_data.get("thread_context_summary", "")
+        competitors_mentioned = input_data.get("competitors_mentioned", [])
+
+        context_section = f"""## PRE-VALIDATED CONTEXT (trust these values)
+- Products mentioned: {products_mentioned if products_mentioned else 'None specific'}
+- {brand_name} mention context: {sabian_context}
+- Author role: {author_role}
+- Competitors mentioned: {competitors_mentioned if competitors_mentioned else 'None'}
+- Thread summary: {thread_summary if thread_summary else 'Not available'}
+"""
+
+        return f"""Analyze this post about {brand_name} for sentiment and intents.
+
+{context_section}
+## POST CONTENT TO ANALYZE:
+\"\"\"{content}\"\"\"
+
+Remember:
+- Sentiment is about {brand_name} ONLY, not overall post tone
+- pain_points/delight_factors only from author's OWN experience
+- Use only values from the valid lists provided
+
+Return JSON only."""
+
+    def analyze_post(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Perform sentiment and intent analysis.
+
+        Args:
+            input_data: Structured data from extraction agent
+
+        Returns:
+            Dictionary with analysis results
+        """
+        try:
+            messages = [
+                SystemMessage(content=self._build_system_prompt()),
+                HumanMessage(content=self._build_user_prompt(input_data))
+            ]
+
+            response = self.llm.invoke(messages)
+            result = self._parse_llm_json_response(response.content)
+
+            # Validate and normalize
+            validated = self._validate_and_normalize(result)
+
+            return {"success": True, **validated}
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"JSON decode error: {e}", "warning")
+            return {
+                "success": False,
+                "error": f"JSON parse error: {str(e)}",
+                "sentiment_level": "neutral",
+                "intents": ["general_discussion"]
+            }
+
+        except Exception as e:
+            self.log_processing(f"Analysis error: {e}", "error")
+            return {"success": False, "error": str(e)}
+
+    def _validate_single(self, value: Any, valid_list: List[str], default: Any = None) -> Any:
+        """Validate single value against list, return canonical form or default."""
+        if value is None:
+            return default
+        if isinstance(value, str):
+            val_lower = value.lower()
+            for v in valid_list:
+                if v.lower() == val_lower:
+                    return v
+        return default
+
+    def _validate_list(self, values: Any, valid_list: List[str]) -> List[str]:
+        """Validate list values, return only valid items in canonical form."""
+        if not values:
+            return []
+        if not isinstance(values, list):
+            values = [values]
+
+        validated = []
+        valid_lower = {v.lower(): v for v in valid_list}
+        for val in values:
+            if isinstance(val, str) and val.lower() in valid_lower:
+                validated.append(valid_lower[val.lower()])
+        return validated
+
+    def _validate_and_normalize(self, result: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate all fields against predefined values and normalize."""
+        v = self._valid_values
+
+        normalized = {
+            # Sentiment
+            "sentiment_level": self._validate_single(
+                result.get("sentiment_level"), v["sentiment_level"], "neutral"
+            ),
+            "emotion_type": self._validate_single(
+                result.get("emotion_type"), v["emotion_type"], None
+            ),
+            "sentiment_confidence": result.get("sentiment_confidence", "medium"),
+            "sarcasm_detected": bool(result.get("sarcasm_detected", False)),
+
+            # Product info
+            "product_attributes": self._validate_list(
+                result.get("product_attributes"), v["product_attributes"]
+            ),
+
+            # Competitors
+            "competitor_products_owned": self._validate_list(
+                result.get("competitor_products_owned"), v["competitors"]
+            ),
+            "comparison_type": self._validate_single(
+                result.get("comparison_type"), v["comparison_type"], None
+            ),
+
+            # Intents
+            "intents": self._validate_list(
+                result.get("intents"), v["intents"]
+            ) or ["general_discussion"],
+
+            # Author journey (null if advising others)
+            "purchase_stage": self._validate_single(
+                result.get("purchase_stage"), v["purchase_stage"], None
+            ),
+            "decision_drivers": self._validate_list(
+                result.get("decision_drivers"), v["decision_drivers"]
+            ),
+
+            # Feedback - both use feedback_aspects
+            "pain_points": self._validate_list(
+                result.get("pain_points"), v["feedback_aspects"]
+            ),
+            "delight_factors": self._validate_list(
+                result.get("delight_factors"), v["feedback_aspects"]
+            ),
+
+            # Notes
+            "analysis_notes": result.get("analysis_notes", ""),
+        }
+
+        # Validate confidence
+        if normalized["sentiment_confidence"] not in ["high", "medium", "low"]:
+            normalized["sentiment_confidence"] = "medium"
+
+        return normalized
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a post through sentiment and intent analysis.
+
+        Args:
+            input_data: Dictionary from extraction agent containing:
+                - cleaned_content: Post text
+                - is_relevant: Relevance determination
+                - products_mentioned: Pre-validated products
+                - sabian_mention_context: How Sabian is discussed
+                - author_role: Author's relationship to Sabian
+                - thread_context_summary: Summarized context
+                - competitors_mentioned: Competitor brands
+
+        Returns:
+            Dictionary with analysis results and original data
+        """
+        try:
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    **input_data
+                }
+
+            # Skip non-relevant posts
+            if not input_data.get("is_relevant", False):
+                return {
+                    "success": True,
+                    "analysis_skipped": True,
+                    "analysis_skip_reason": "Post marked as not relevant",
+                    "sentiment_level": None,
+                    "emotion_type": None,
+                    "sentiment_confidence": None,
+                    "sarcasm_detected": False,
+                    "product_attributes": [],
+                    "competitor_products_owned": [],
+                    "comparison_type": None,
+                    "intents": [],
+                    "purchase_stage": None,
+                    "decision_drivers": [],
+                    "pain_points": [],
+                    "delight_factors": [],
+                    "analysis_notes": "",
+                    **input_data
+                }
+
+            # Skip non-English posts (should already be filtered, but double-check)
+            if not input_data.get("is_english", True):
+                return {
+                    "success": True,
+                    "analysis_skipped": True,
+                    "analysis_skip_reason": f"Non-English: {input_data.get('detected_language')}",
+                    "sentiment_level": None,
+                    "emotion_type": None,
+                    "intents": [],
+                    **input_data
+                }
+
+            # Perform analysis
+            analysis_result = self.analyze_post(input_data)
+
+            result = {
+                **input_data,
+                **analysis_result,
+                "analysis_skipped": False
+            }
+
+            self.log_processing(
+                f"Analyzed: sentiment={result.get('sentiment_level')}, "
+                f"intents={result.get('intents')}, "
+                f"pain_points={result.get('pain_points')}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "sentiment analysis")

processing_brand_sentiment/workflow/comment_orchestrator.py

@@ -0,0 +1,558 @@
+"""
+Comment Analysis Workflow Orchestrator using LangGraph.
+
+Coordinates the 4-agent pipeline for social media comments:
+1. CommentPreprocessorAgent - Plain text cleaning, keyword detection (no LLM)
+2. SabianRelevanceExtractionAgent - Relevance + fact extraction (LLM #1) [shared]
+3. SabianSentimentAnalyzerAgent - Deep sentiment analysis (LLM #2) [shared]
+4. OutputValidatorAgent - Rule-based validation (no LLM) [shared]
+
+Architecture v4.0:
+- Same analysis pipeline as forums, different preprocessing and state
+- Plain text input (no HTML parsing)
+- Context from social media content metadata and parent comments
+- Comment-specific identifiers (comment_sk, comment_id, platform, etc.)
+"""
+
+from typing import Dict, Any, List, TypedDict, Annotated, Optional
+import operator
+import json
+import os
+from langgraph.graph import StateGraph, END
+import logging
+
+from .agents.comment_preprocessor_agent import CommentPreprocessorAgent
+from .agents.sabian_relevance_extraction_agent import SabianRelevanceExtractionAgent
+from .agents.sabian_sentiment_analyzer_agent import SabianSentimentAnalyzerAgent
+from .agents.output_validator_agent import OutputValidatorAgent
+
+logger = logging.getLogger(__name__)
+
+
+class CommentAnalysisState(TypedDict):
+    """
+    State definition for the comment analysis workflow v4.0.
+
+    Uses comment-specific identifiers but shares the same analysis fields
+    as the forum workflow for consistent output.
+    """
+    # ============== Source Identifiers (Comment-specific) ==============
+    comment_sk: int
+    comment_id: str
+    platform: str
+    comment_timestamp: Any
+    author_name: str
+    author_id: str
+    parent_comment_id: str
+    parent_comment_text: str
+
+    # Content metadata
+    content_sk: int
+    content_id: str
+    content_description: str
+    content_title: str
+    channel_sk: int
+    channel_name: str
+    channel_display_name: str
+
+    # ============== Original Content ==============
+    comment_text: str
+    original_text: str
+
+    # ============== Preprocessor Output ==============
+    cleaned_content: str
+    quoted_content: str
+    has_quote: bool
+    quoted_author: str
+    raw_thread_context: str  # Comment context (reuses field name for agent compatibility)
+    is_empty: bool
+
+    # Language detection
+    detected_language: str
+    language_code: str
+    is_english: bool
+    language_confidence: str
+    language_detection_skipped: bool
+
+    # Preliminary relevance (keyword-based)
+    preliminary_relevant: bool
+    needs_relevance_validation: bool
+    relevance_keywords_found: List[str]
+    relevance_type: str
+    has_primary_keywords: bool
+
+    # Initial detections
+    products_detected: List[str]
+    competitors_detected: List[str]
+
+    # ============== Extraction Agent Output ==============
+    is_relevant: bool
+    relevance_confidence: str
+    relevance_reason: str
+    extraction_performed: bool
+
+    # Extracted facts
+    products_mentioned: List[str]
+    sabian_mention_context: str
+    author_role: str
+    competitors_mentioned: List[str]
+    thread_context_summary: str
+
+    # ============== Sentiment Analyzer Output ==============
+    sentiment_level: str
+    emotion_type: str
+    sentiment_confidence: str
+    sarcasm_detected: bool
+
+    # Product information
+    product_attributes: List[str]
+
+    # Competitive intelligence
+    competitor_products_owned: List[str]
+    comparison_type: str
+
+    # Customer journey (AUTHOR PERSPECTIVE ONLY)
+    intents: List[str]
+    purchase_stage: str
+    decision_drivers: List[str]
+    pain_points: List[str]
+    delight_factors: List[str]
+
+    # Analysis notes
+    analysis_notes: str
+    analysis_skipped: bool
+    analysis_skip_reason: str
+
+    # ============== Validator Output ==============
+    validation_passed: bool
+    validation_errors: List[str]
+    validation_warnings: List[str]
+    validation_flags: List[str]
+    processing_status: str
+
+    # ============== Processing Metadata ==============
+    processing_errors: Annotated[List[str], operator.add]
+    success: bool
+
+
+class CommentAnalysisWorkflow:
+    """
+    LangGraph-based workflow for comment brand sentiment analysis v4.0.
+
+    Pipeline:
+    1. Comment Preprocessor (no LLM) - plain text, comment context
+    2. Relevance & Extraction Agent (LLM #1) - shared with forums
+    3. Sentiment Analyzer Agent (LLM #2) - shared with forums
+    4. Output Validator (no LLM) - shared with forums
+    """
+
+    def __init__(
+        self,
+        workflow_config: Dict[str, Any],
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any],
+        api_key: str
+    ):
+        """
+        Initialize the workflow with agents and configuration.
+
+        Args:
+            workflow_config: Workflow and agent configuration
+            brand_config: Brand-specific configuration
+            analysis_categories: Analysis category definitions
+            api_key: OpenAI API key
+        """
+        self.workflow_config = workflow_config
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+        self.api_key = api_key
+
+        # Initialize agents
+        self._init_agents()
+
+        # Build the workflow graph
+        self.workflow = self._build_workflow()
+
+        logger.info("CommentAnalysisWorkflow v4.0 initialized successfully")
+
+    def _init_agents(self) -> None:
+        """Initialize all agents with their configurations."""
+        agents_config = self.workflow_config.get("agents", {})
+
+        # 1. Comment Preprocessor Agent (no LLM) - comment-specific
+        preprocessor_config = agents_config.get("preprocessor", {})
+        self.preprocessor = CommentPreprocessorAgent(
+            preprocessor_config,
+            self.brand_config
+        )
+
+        # 2. Relevance & Extraction Agent (LLM #1) - shared with forums
+        extraction_config = agents_config.get("relevance_extraction",
+            agents_config.get("relevance_validator", {})
+        )
+        self.extraction_agent = SabianRelevanceExtractionAgent(
+            extraction_config,
+            self.api_key,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        # 3. Sentiment Analyzer Agent (LLM #2) - shared with forums
+        analyzer_config = agents_config.get("sentiment_analyzer",
+            agents_config.get("brand_analyzer", {})
+        )
+        self.sentiment_analyzer = SabianSentimentAnalyzerAgent(
+            analyzer_config,
+            self.api_key,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        # 4. Output Validator Agent (no LLM) - shared with forums
+        validator_config = agents_config.get("output_validator", {})
+        self.output_validator = OutputValidatorAgent(
+            validator_config,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        logger.info("All 4 agents initialized for comment processing")
+
+    def _build_workflow(self) -> StateGraph:
+        """
+        Build the LangGraph workflow.
+
+        Flow:
+        preprocessing -> extraction -> (analysis if relevant) -> validation -> END
+
+        Returns:
+            Compiled StateGraph workflow
+        """
+        workflow = StateGraph(CommentAnalysisState)
+
+        # Add nodes
+        workflow.add_node("preprocessing", self._preprocessing_node)
+        workflow.add_node("extraction", self._extraction_node)
+        workflow.add_node("analysis", self._analysis_node)
+        workflow.add_node("validation", self._validation_node)
+
+        # Set entry point
+        workflow.set_entry_point("preprocessing")
+
+        # Define edges
+        workflow.add_conditional_edges(
+            "preprocessing",
+            self._route_after_preprocessing,
+            {
+                "extract": "extraction",
+                "skip_to_validation": "validation"
+            }
+        )
+
+        workflow.add_conditional_edges(
+            "extraction",
+            self._route_after_extraction,
+            {
+                "analyze": "analysis",
+                "skip_to_validation": "validation"
+            }
+        )
+
+        workflow.add_edge("analysis", "validation")
+        workflow.add_edge("validation", END)
+
+        return workflow.compile()
+
+    def _preprocessing_node(self, state: CommentAnalysisState) -> CommentAnalysisState:
+        """
+        Preprocessing node: Plain text cleaning, language detection, keyword check.
+        """
+        try:
+            input_data = {
+                "comment_sk": state.get("comment_sk"),
+                "comment_text": state.get("comment_text", ""),
+                "content_title": state.get("content_title"),
+                "content_description": state.get("content_description"),
+                "parent_comment_text": state.get("parent_comment_text")
+            }
+
+            result = self.preprocessor.process(input_data)
+
+            if result.get("success", False):
+                # Content
+                state["cleaned_content"] = result.get("cleaned_content", "")
+                state["quoted_content"] = result.get("quoted_content")
+                state["has_quote"] = result.get("has_quote", False)
+                state["quoted_author"] = result.get("quoted_author")
+                state["raw_thread_context"] = result.get("raw_thread_context", "")
+                state["is_empty"] = result.get("is_empty", False)
+                state["original_text"] = result.get("original_text", state.get("comment_text", ""))
+
+                # Language
+                state["detected_language"] = result.get("detected_language", "English")
+                state["language_code"] = result.get("language_code", "en")
+                state["is_english"] = result.get("is_english", True)
+                state["language_confidence"] = result.get("language_confidence", "low")
+                state["language_detection_skipped"] = result.get("language_detection_skipped", False)
+
+                # Relevance
+                state["preliminary_relevant"] = result.get("preliminary_relevant", False)
+                state["needs_relevance_validation"] = result.get("needs_relevance_validation", False)
+                state["relevance_keywords_found"] = result.get("relevance_keywords_found", [])
+                state["relevance_type"] = result.get("relevance_type", "none")
+                state["has_primary_keywords"] = result.get("has_primary_keywords", False)
+
+                # Detections
+                state["products_detected"] = result.get("products_detected", [])
+                state["competitors_detected"] = result.get("competitors_detected", [])
+
+                state["success"] = True
+            else:
+                error_msg = f"Preprocessing failed: {result.get('error', 'Unknown error')}"
+                state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+                state["success"] = False
+
+            logger.debug(f"Preprocessing complete for comment {state.get('comment_sk')}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Preprocessing node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["success"] = False
+            return state
+
+    def _extraction_node(self, state: CommentAnalysisState) -> CommentAnalysisState:
+        """
+        Extraction node: LLM-based relevance validation and fact extraction.
+        Reuses the same extraction agent as forums.
+        """
+        try:
+            input_data = {
+                "cleaned_content": state.get("cleaned_content", ""),
+                "quoted_content": state.get("quoted_content"),
+                "raw_thread_context": state.get("raw_thread_context", ""),
+                "relevance_keywords_found": state.get("relevance_keywords_found", []),
+                "preliminary_relevant": state.get("preliminary_relevant", False),
+                "needs_relevance_validation": state.get("needs_relevance_validation", True),
+                "products_detected": state.get("products_detected", []),
+                "competitors_detected": state.get("competitors_detected", []),
+                "is_english": state.get("is_english", True),
+                "detected_language": state.get("detected_language", "English")
+            }
+
+            result = self.extraction_agent.process(input_data)
+
+            # Update state with extraction results
+            state["is_relevant"] = result.get("is_relevant", False)
+            state["relevance_confidence"] = result.get("relevance_confidence", "low")
+            state["relevance_reason"] = result.get("relevance_reason", "")
+            state["extraction_performed"] = result.get("extraction_performed", True)
+
+            # Extracted facts
+            state["products_mentioned"] = result.get("products_mentioned", [])
+            state["sabian_mention_context"] = result.get("sabian_mention_context")
+            state["author_role"] = result.get("author_role", "unknown")
+            state["competitors_mentioned"] = result.get("competitors_mentioned", [])
+            state["thread_context_summary"] = result.get("thread_context_summary", "")
+
+            if not result.get("success", False) and result.get("error"):
+                state["processing_errors"] = state.get("processing_errors", []) + [result["error"]]
+
+            logger.debug(f"Extraction complete: is_relevant={state['is_relevant']}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Extraction node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["is_relevant"] = False
+            state["relevance_confidence"] = "low"
+            return state
+
+    def _analysis_node(self, state: CommentAnalysisState) -> CommentAnalysisState:
+        """
+        Analysis node: Deep sentiment and intent analysis for relevant comments.
+        Reuses the same sentiment analyzer as forums.
+        """
+        try:
+            input_data = {
+                "cleaned_content": state.get("cleaned_content", ""),
+                "is_relevant": state.get("is_relevant", True),
+                "is_english": state.get("is_english", True),
+                "detected_language": state.get("detected_language", "English"),
+                "products_mentioned": state.get("products_mentioned", []),
+                "sabian_mention_context": state.get("sabian_mention_context"),
+                "author_role": state.get("author_role", "unknown"),
+                "competitors_mentioned": state.get("competitors_mentioned", []),
+                "thread_context_summary": state.get("thread_context_summary", "")
+            }
+
+            result = self.sentiment_analyzer.process(input_data)
+
+            if result.get("success", False):
+                # Sentiment
+                state["sentiment_level"] = result.get("sentiment_level")
+                state["emotion_type"] = result.get("emotion_type")
+                state["sentiment_confidence"] = result.get("sentiment_confidence", "medium")
+                state["sarcasm_detected"] = result.get("sarcasm_detected", False)
+
+                # Products
+                state["product_attributes"] = result.get("product_attributes", [])
+
+                # Competitive
+                state["competitor_products_owned"] = result.get("competitor_products_owned", [])
+                state["comparison_type"] = result.get("comparison_type")
+
+                # Journey
+                state["intents"] = result.get("intents", [])
+                state["purchase_stage"] = result.get("purchase_stage")
+                state["decision_drivers"] = result.get("decision_drivers", [])
+                state["pain_points"] = result.get("pain_points", [])
+                state["delight_factors"] = result.get("delight_factors", [])
+
+                # Notes
+                state["analysis_notes"] = result.get("analysis_notes", "")
+                state["analysis_skipped"] = result.get("analysis_skipped", False)
+                state["analysis_skip_reason"] = result.get("analysis_skip_reason", "")
+            else:
+                error_msg = f"Analysis failed: {result.get('error', 'Unknown error')}"
+                state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+
+            logger.debug(f"Analysis complete for comment {state.get('comment_sk')}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Analysis node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            return state
+
+    def _validation_node(self, state: CommentAnalysisState) -> CommentAnalysisState:
+        """
+        Validation node: Rule-based validation and anomaly detection.
+        Reuses the same output validator as forums.
+        """
+        try:
+            result = self.output_validator.process(dict(state))
+
+            state["validation_passed"] = result.get("validation_passed", True)
+            state["validation_errors"] = result.get("validation_errors", [])
+            state["validation_warnings"] = result.get("validation_warnings", [])
+            state["validation_flags"] = result.get("validation_flags", [])
+            state["processing_status"] = result.get("processing_status", "completed")
+
+            # Set overall success
+            has_errors = len(state.get("processing_errors", [])) > 0
+            state["success"] = not has_errors or state.get("is_relevant") is not None
+
+            logger.debug(f"Validation complete: status={state['processing_status']}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Validation node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["validation_passed"] = False
+            state["processing_status"] = "validation_failed"
+            state["success"] = False
+            return state
+
+    def _route_after_preprocessing(self, state: CommentAnalysisState) -> str:
+        """Determine routing after preprocessing."""
+        if state.get("is_empty", False):
+            state["is_relevant"] = False
+            state["relevance_reason"] = "Empty content"
+            return "skip_to_validation"
+
+        if not state.get("is_english", True):
+            state["is_relevant"] = False
+            state["relevance_reason"] = f"Non-English: {state.get('detected_language')}"
+            return "skip_to_validation"
+
+        if (not state.get("preliminary_relevant", False) and
+            not state.get("needs_relevance_validation", False)):
+            state["is_relevant"] = False
+            state["relevance_reason"] = "No relevant keywords found"
+            return "skip_to_validation"
+
+        return "extract"
+
+    def _route_after_extraction(self, state: CommentAnalysisState) -> str:
+        """Determine routing after extraction."""
+        if state.get("is_relevant", False):
+            return "analyze"
+        return "skip_to_validation"
+
+    def process_comment(self, comment_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a single social media comment through the workflow.
+
+        Args:
+            comment_data: Dictionary containing comment data
+
+        Returns:
+            Dictionary with processed results
+        """
+        try:
+            initial_state = {
+                # Comment identifiers
+                "comment_sk": comment_data.get("comment_sk"),
+                "comment_id": comment_data.get("comment_id"),
+                "platform": comment_data.get("platform"),
+                "comment_timestamp": comment_data.get("comment_timestamp"),
+                "author_name": comment_data.get("author_name"),
+                "author_id": comment_data.get("author_id"),
+                "parent_comment_id": comment_data.get("parent_comment_id"),
+                "parent_comment_text": comment_data.get("parent_comment_text"),
+
+                # Content metadata
+                "content_sk": comment_data.get("content_sk"),
+                "content_id": comment_data.get("content_id"),
+                "content_description": comment_data.get("content_description"),
+                "content_title": comment_data.get("content_title"),
+                "channel_sk": comment_data.get("channel_sk"),
+                "channel_name": comment_data.get("channel_name"),
+                "channel_display_name": comment_data.get("channel_display_name"),
+
+                # Comment text
+                "comment_text": comment_data.get("comment_text", ""),
+
+                # Processing metadata
+                "processing_errors": [],
+                "success": True
+            }
+
+            final_state = self.workflow.invoke(initial_state)
+
+            return dict(final_state)
+
+        except Exception as e:
+            logger.error(f"Workflow execution error: {str(e)}")
+            return {
+                **comment_data,
+                "success": False,
+                "processing_errors": [str(e)],
+                "processing_status": "workflow_error"
+            }
+
+    def process_batch(self, comments: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Process a batch of social media comments.
+
+        Args:
+            comments: List of comment dictionaries
+
+        Returns:
+            List of processed comment dictionaries
+        """
+        results = []
+        total = len(comments)
+
+        for idx, comment in enumerate(comments, 1):
+            logger.info(f"Processing comment {idx}/{total} (SK: {comment.get('comment_sk')})")
+            result = self.process_comment(comment)
+            results.append(result)
+
+        logger.info(f"Batch processing complete: {total} comments processed")
+        return results

processing_brand_sentiment/workflow/orchestrator.py

@@ -0,0 +1,551 @@
+"""
+Brand Analysis Workflow Orchestrator using LangGraph.
+
+Coordinates the 4-agent pipeline:
+1. ContentPreprocessorAgent - HTML parsing, cleaning, keyword detection (no LLM)
+2. SabianRelevanceExtractionAgent - Relevance + fact extraction (LLM #1)
+3. SabianSentimentAnalyzerAgent - Deep sentiment analysis (LLM #2)
+4. OutputValidatorAgent - Rule-based validation (no LLM)
+
+Architecture v4.0:
+- Separation of concerns: extraction vs analysis
+- Strict validation at every step
+- Structured data flow between agents
+- Conservative relevance determination
+"""
+
+from typing import Dict, Any, List, TypedDict, Annotated, Optional
+import operator
+import json
+import os
+from langgraph.graph import StateGraph, END
+import logging
+
+from .agents.content_preprocessor_agent import ContentPreprocessorAgent
+from .agents.sabian_relevance_extraction_agent import SabianRelevanceExtractionAgent
+from .agents.sabian_sentiment_analyzer_agent import SabianSentimentAnalyzerAgent
+from .agents.output_validator_agent import OutputValidatorAgent
+
+logger = logging.getLogger(__name__)
+
+
+class BrandAnalysisState(TypedDict):
+    """
+    State definition for the brand analysis workflow v4.0.
+
+    This state flows through all agents, accumulating data at each step.
+    """
+    # ============== Source Identifiers ==============
+    post_id: int
+    thread_id: int
+    post_author_id: int
+
+    # ============== Original Content ==============
+    post_content: str
+    original_content: str
+
+    # ============== Thread Context ==============
+    thread_title: str
+    thread_first_post: str
+    thread_started_at: Any
+    category_title: str
+    category_topic: str
+
+    # ============== Timestamps ==============
+    post_created_at: Any
+
+    # ============== Preprocessor Output ==============
+    cleaned_content: str
+    quoted_content: str
+    has_quote: bool
+    quoted_author: str
+    raw_thread_context: str  # Raw context for extraction agent
+    is_empty: bool
+
+    # Language detection
+    detected_language: str
+    language_code: str
+    is_english: bool
+    language_confidence: str
+    language_detection_skipped: bool
+
+    # Preliminary relevance (keyword-based)
+    preliminary_relevant: bool
+    needs_relevance_validation: bool
+    relevance_keywords_found: List[str]
+    relevance_type: str
+    has_primary_keywords: bool
+
+    # Initial detections
+    products_detected: List[str]
+    competitors_detected: List[str]
+
+    # ============== Extraction Agent Output ==============
+    is_relevant: bool
+    relevance_confidence: str
+    relevance_reason: str
+    extraction_performed: bool
+
+    # Extracted facts
+    products_mentioned: List[str]
+    sabian_mention_context: str  # primary_focus, significant_mention, casual_mention, comparison_context
+    author_role: str  # current_owner, past_owner, potential_buyer, never_owned, unknown
+    competitors_mentioned: List[str]
+    thread_context_summary: str  # NEW: Summarized context for storage and analysis
+
+    # ============== Sentiment Analyzer Output ==============
+    sentiment_level: str
+    emotion_type: str
+    sentiment_confidence: str
+    sarcasm_detected: bool
+
+    # Product information
+    product_attributes: List[str]
+
+    # Competitive intelligence
+    competitor_products_owned: List[str]
+    comparison_type: str
+
+    # Customer journey (AUTHOR PERSPECTIVE ONLY)
+    intents: List[str]
+    purchase_stage: str
+    decision_drivers: List[str]
+    pain_points: List[str]
+    delight_factors: List[str]
+
+    # Analysis notes
+    analysis_notes: str
+    analysis_skipped: bool
+    analysis_skip_reason: str
+
+    # ============== Validator Output ==============
+    validation_passed: bool
+    validation_errors: List[str]
+    validation_warnings: List[str]
+    validation_flags: List[str]
+    processing_status: str  # completed, completed_with_flags, validation_failed
+
+    # ============== Processing Metadata ==============
+    processing_errors: Annotated[List[str], operator.add]
+    success: bool
+
+
+class BrandAnalysisWorkflow:
+    """
+    LangGraph-based workflow for brand sentiment analysis v4.0.
+
+    Pipeline:
+    1. Content Preprocessor (no LLM)
+    2. Relevance & Extraction Agent (LLM #1)
+    3. Sentiment Analyzer Agent (LLM #2) - only for relevant posts
+    4. Output Validator (no LLM)
+    """
+
+    def __init__(
+        self,
+        workflow_config: Dict[str, Any],
+        brand_config: Dict[str, Any],
+        analysis_categories: Dict[str, Any],
+        api_key: str
+    ):
+        """
+        Initialize the workflow with agents and configuration.
+
+        Args:
+            workflow_config: Workflow and agent configuration
+            brand_config: Brand-specific configuration
+            analysis_categories: Analysis category definitions
+            api_key: OpenAI API key
+        """
+        self.workflow_config = workflow_config
+        self.brand_config = brand_config
+        self.analysis_categories = analysis_categories
+        self.api_key = api_key
+
+        # Initialize agents
+        self._init_agents()
+
+        # Build the workflow graph
+        self.workflow = self._build_workflow()
+
+        logger.info("BrandAnalysisWorkflow v4.0 initialized successfully")
+
+    def _init_agents(self) -> None:
+        """Initialize all agents with their configurations."""
+        agents_config = self.workflow_config.get("agents", {})
+
+        # 1. Content Preprocessor Agent (no LLM)
+        preprocessor_config = agents_config.get("preprocessor", {})
+        self.preprocessor = ContentPreprocessorAgent(
+            preprocessor_config,
+            self.brand_config
+        )
+
+        # 2. Relevance & Extraction Agent (LLM #1)
+        extraction_config = agents_config.get("relevance_extraction",
+            agents_config.get("relevance_validator", {})  # Fallback to old config
+        )
+        self.extraction_agent = SabianRelevanceExtractionAgent(
+            extraction_config,
+            self.api_key,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        # 3. Sentiment Analyzer Agent (LLM #2)
+        analyzer_config = agents_config.get("sentiment_analyzer",
+            agents_config.get("brand_analyzer", {})  # Fallback to old config
+        )
+        self.sentiment_analyzer = SabianSentimentAnalyzerAgent(
+            analyzer_config,
+            self.api_key,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        # 4. Output Validator Agent (no LLM)
+        validator_config = agents_config.get("output_validator", {})
+        self.output_validator = OutputValidatorAgent(
+            validator_config,
+            self.brand_config,
+            self.analysis_categories
+        )
+
+        logger.info("All 4 agents initialized")
+
+    def _build_workflow(self) -> StateGraph:
+        """
+        Build the LangGraph workflow.
+
+        Flow:
+        preprocessing -> extraction -> (analysis if relevant) -> validation -> END
+
+        Returns:
+            Compiled StateGraph workflow
+        """
+        workflow = StateGraph(BrandAnalysisState)
+
+        # Add nodes
+        workflow.add_node("preprocessing", self._preprocessing_node)
+        workflow.add_node("extraction", self._extraction_node)
+        workflow.add_node("analysis", self._analysis_node)
+        workflow.add_node("validation", self._validation_node)
+
+        # Set entry point
+        workflow.set_entry_point("preprocessing")
+
+        # Define edges
+        # Preprocessing -> conditional routing
+        workflow.add_conditional_edges(
+            "preprocessing",
+            self._route_after_preprocessing,
+            {
+                "extract": "extraction",
+                "skip_to_validation": "validation"
+            }
+        )
+
+        # Extraction -> conditional routing
+        workflow.add_conditional_edges(
+            "extraction",
+            self._route_after_extraction,
+            {
+                "analyze": "analysis",
+                "skip_to_validation": "validation"
+            }
+        )
+
+        # Analysis -> validation
+        workflow.add_edge("analysis", "validation")
+
+        # Validation -> END
+        workflow.add_edge("validation", END)
+
+        return workflow.compile()
+
+    def _preprocessing_node(self, state: BrandAnalysisState) -> BrandAnalysisState:
+        """
+        Preprocessing node: HTML parsing, cleaning, language detection, keyword check.
+        """
+        try:
+            input_data = {
+                "post_id": state.get("post_id"),
+                "post_content": state.get("post_content", ""),
+                "thread_title": state.get("thread_title"),
+                "thread_first_post": state.get("thread_first_post"),
+                "category_title": state.get("category_title"),
+                "category_topic": state.get("category_topic")
+            }
+
+            result = self.preprocessor.process(input_data)
+
+            if result.get("success", False):
+                # Content
+                state["cleaned_content"] = result.get("cleaned_content", "")
+                state["quoted_content"] = result.get("quoted_content")
+                state["has_quote"] = result.get("has_quote", False)
+                state["quoted_author"] = result.get("quoted_author")
+                state["raw_thread_context"] = result.get("raw_thread_context", "")
+                state["is_empty"] = result.get("is_empty", False)
+                state["original_content"] = result.get("original_content", state.get("post_content", ""))
+
+                # Language
+                state["detected_language"] = result.get("detected_language", "English")
+                state["language_code"] = result.get("language_code", "en")
+                state["is_english"] = result.get("is_english", True)
+                state["language_confidence"] = result.get("language_confidence", "low")
+                state["language_detection_skipped"] = result.get("language_detection_skipped", False)
+
+                # Relevance
+                state["preliminary_relevant"] = result.get("preliminary_relevant", False)
+                state["needs_relevance_validation"] = result.get("needs_relevance_validation", False)
+                state["relevance_keywords_found"] = result.get("relevance_keywords_found", [])
+                state["relevance_type"] = result.get("relevance_type", "none")
+                state["has_primary_keywords"] = result.get("has_primary_keywords", False)
+
+                # Detections
+                state["products_detected"] = result.get("products_detected", [])
+                state["competitors_detected"] = result.get("competitors_detected", [])
+
+                state["success"] = True
+            else:
+                error_msg = f"Preprocessing failed: {result.get('error', 'Unknown error')}"
+                state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+                state["success"] = False
+
+            logger.debug(f"Preprocessing complete for post {state.get('post_id')}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Preprocessing node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["success"] = False
+            return state
+
+    def _extraction_node(self, state: BrandAnalysisState) -> BrandAnalysisState:
+        """
+        Extraction node: LLM-based relevance validation and fact extraction.
+        """
+        try:
+            input_data = {
+                "cleaned_content": state.get("cleaned_content", ""),
+                "quoted_content": state.get("quoted_content"),
+                "raw_thread_context": state.get("raw_thread_context", ""),
+                "relevance_keywords_found": state.get("relevance_keywords_found", []),
+                "preliminary_relevant": state.get("preliminary_relevant", False),
+                "needs_relevance_validation": state.get("needs_relevance_validation", True),
+                "products_detected": state.get("products_detected", []),
+                "competitors_detected": state.get("competitors_detected", []),
+                "is_english": state.get("is_english", True),
+                "detected_language": state.get("detected_language", "English")
+            }
+
+            result = self.extraction_agent.process(input_data)
+
+            # Update state with extraction results
+            state["is_relevant"] = result.get("is_relevant", False)
+            state["relevance_confidence"] = result.get("relevance_confidence", "low")
+            state["relevance_reason"] = result.get("relevance_reason", "")
+            state["extraction_performed"] = result.get("extraction_performed", True)
+
+            # Extracted facts
+            state["products_mentioned"] = result.get("products_mentioned", [])
+            state["sabian_mention_context"] = result.get("sabian_mention_context")
+            state["author_role"] = result.get("author_role", "unknown")
+            state["competitors_mentioned"] = result.get("competitors_mentioned", [])
+            state["thread_context_summary"] = result.get("thread_context_summary", "")
+
+            if not result.get("success", False) and result.get("error"):
+                state["processing_errors"] = state.get("processing_errors", []) + [result["error"]]
+
+            logger.debug(f"Extraction complete: is_relevant={state['is_relevant']}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Extraction node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["is_relevant"] = False
+            state["relevance_confidence"] = "low"
+            return state
+
+    def _analysis_node(self, state: BrandAnalysisState) -> BrandAnalysisState:
+        """
+        Analysis node: Deep sentiment and intent analysis for relevant posts.
+        """
+        try:
+            input_data = {
+                "cleaned_content": state.get("cleaned_content", ""),
+                "is_relevant": state.get("is_relevant", True),
+                "is_english": state.get("is_english", True),
+                "detected_language": state.get("detected_language", "English"),
+                "products_mentioned": state.get("products_mentioned", []),
+                "sabian_mention_context": state.get("sabian_mention_context"),
+                "author_role": state.get("author_role", "unknown"),
+                "competitors_mentioned": state.get("competitors_mentioned", []),
+                "thread_context_summary": state.get("thread_context_summary", "")
+            }
+
+            result = self.sentiment_analyzer.process(input_data)
+
+            if result.get("success", False):
+                # Sentiment
+                state["sentiment_level"] = result.get("sentiment_level")
+                state["emotion_type"] = result.get("emotion_type")
+                state["sentiment_confidence"] = result.get("sentiment_confidence", "medium")
+                state["sarcasm_detected"] = result.get("sarcasm_detected", False)
+
+                # Products
+                state["product_attributes"] = result.get("product_attributes", [])
+
+                # Competitive
+                state["competitor_products_owned"] = result.get("competitor_products_owned", [])
+                state["comparison_type"] = result.get("comparison_type")
+
+                # Journey
+                state["intents"] = result.get("intents", [])
+                state["purchase_stage"] = result.get("purchase_stage")
+                state["decision_drivers"] = result.get("decision_drivers", [])
+                state["pain_points"] = result.get("pain_points", [])
+                state["delight_factors"] = result.get("delight_factors", [])
+
+                # Notes
+                state["analysis_notes"] = result.get("analysis_notes", "")
+                state["analysis_skipped"] = result.get("analysis_skipped", False)
+                state["analysis_skip_reason"] = result.get("analysis_skip_reason", "")
+            else:
+                error_msg = f"Analysis failed: {result.get('error', 'Unknown error')}"
+                state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+
+            logger.debug(f"Analysis complete for post {state.get('post_id')}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Analysis node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            return state
+
+    def _validation_node(self, state: BrandAnalysisState) -> BrandAnalysisState:
+        """
+        Validation node: Rule-based validation and anomaly detection.
+        """
+        try:
+            result = self.output_validator.process(dict(state))
+
+            state["validation_passed"] = result.get("validation_passed", True)
+            state["validation_errors"] = result.get("validation_errors", [])
+            state["validation_warnings"] = result.get("validation_warnings", [])
+            state["validation_flags"] = result.get("validation_flags", [])
+            state["processing_status"] = result.get("processing_status", "completed")
+
+            # Set overall success
+            has_errors = len(state.get("processing_errors", [])) > 0
+            state["success"] = not has_errors or state.get("is_relevant") is not None
+
+            logger.debug(f"Validation complete: status={state['processing_status']}")
+            return state
+
+        except Exception as e:
+            error_msg = f"Validation node error: {str(e)}"
+            logger.error(error_msg)
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["validation_passed"] = False
+            state["processing_status"] = "validation_failed"
+            state["success"] = False
+            return state
+
+    def _route_after_preprocessing(self, state: BrandAnalysisState) -> str:
+        """
+        Determine routing after preprocessing.
+        """
+        # If empty content, skip to validation
+        if state.get("is_empty", False):
+            state["is_relevant"] = False
+            state["relevance_reason"] = "Empty content"
+            return "skip_to_validation"
+
+        # If not English, skip to validation
+        if not state.get("is_english", True):
+            state["is_relevant"] = False
+            state["relevance_reason"] = f"Non-English: {state.get('detected_language')}"
+            return "skip_to_validation"
+
+        # If no keywords found and no need for validation, skip
+        if (not state.get("preliminary_relevant", False) and
+            not state.get("needs_relevance_validation", False)):
+            state["is_relevant"] = False
+            state["relevance_reason"] = "No relevant keywords found"
+            return "skip_to_validation"
+
+        # Otherwise, go to extraction
+        return "extract"
+
+    def _route_after_extraction(self, state: BrandAnalysisState) -> str:
+        """
+        Determine routing after extraction.
+        """
+        if state.get("is_relevant", False):
+            return "analyze"
+        return "skip_to_validation"
+
+    def process_post(self, post_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a single forum post through the workflow.
+
+        Args:
+            post_data: Dictionary containing post data
+
+        Returns:
+            Dictionary with processed results
+        """
+        try:
+            initial_state = {
+                "post_id": post_data.get("post_id"),
+                "thread_id": post_data.get("thread_id"),
+                "post_author_id": post_data.get("post_author_id"),
+                "post_content": post_data.get("post_content", ""),
+                "thread_title": post_data.get("thread_title"),
+                "thread_first_post": post_data.get("thread_first_post"),
+                "thread_started_at": post_data.get("thread_started_at"),
+                "category_title": post_data.get("category_title"),
+                "category_topic": post_data.get("category_topic"),
+                "post_created_at": post_data.get("post_created_at"),
+                "processing_errors": [],
+                "success": True
+            }
+
+            final_state = self.workflow.invoke(initial_state)
+
+            return dict(final_state)
+
+        except Exception as e:
+            logger.error(f"Workflow execution error: {str(e)}")
+            return {
+                **post_data,
+                "success": False,
+                "processing_errors": [str(e)],
+                "processing_status": "workflow_error"
+            }
+
+    def process_batch(self, posts: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Process a batch of forum posts.
+
+        Args:
+            posts: List of post dictionaries
+
+        Returns:
+            List of processed post dictionaries
+        """
+        results = []
+        total = len(posts)
+
+        for idx, post in enumerate(posts, 1):
+            logger.info(f"Processing post {idx}/{total} (ID: {post.get('post_id')})")
+            result = self.process_post(post)
+            results.append(result)
+
+        logger.info(f"Batch processing complete: {total} posts processed")
+        return results

processing_comments/.dockerignore

@@ -0,0 +1,8 @@
+**/__pycache__/
+**/*.pyc
+.git
+.gitignore
+.env
+*.log
+dist
+build

processing_comments/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

processing_comments/README.md

@@ -0,0 +1,726 @@
+# Comment Processing with Agentic Workflow
+
+A scalable, modular system for processing comments from multiple data sources using OpenAI API, LangChain, and LangGraph. The system performs language detection, translation, and context-aware sentiment analysis using an agentic workflow architecture.
+
+## Data Sources Supported
+
+- **Social Media Comments**: External platforms (Facebook, Instagram, YouTube, etc.)
+- **Musora Internal Comments**: Comments from Musora internal applications
+- **Extensible Architecture**: Easily add new data sources via configuration
+
+## Features
+
+- **Multi-Source Support**: Process comments from multiple data sources with a single codebase
+- **Configuration-Driven**: Add new data sources without code changes
+- **Parent Comment Context**: Automatically includes parent comment text for reply analysis
+- **Modular Agent Architecture**: Extensible base classes for easy addition of new agents
+- **Language Detection**: Hybrid approach using lingua library for fast English detection, with LLM fallback for non-English languages
+- **Translation**: High-quality translation for non-English comments using OpenAI models
+- **Context-Aware Sentiment Analysis**:
+  - Uses content description for context
+  - Includes parent comment text when analyzing replies
+  - Multi-label intent classification
+- **LangGraph Workflow**: Flexible graph-based orchestration of agent operations
+- **Snowflake Integration**: Seamless data fetching and storage with source-specific tables
+- **Parallel Processing**: Multiprocessing support for high-performance batch processing
+- **Dynamic Batch Sizing**: Intelligent batch size calculation based on workload and available resources
+- **Independent Batch Execution**: Each batch processes and stores results independently
+- **Comprehensive Logging**: Detailed logging for monitoring and debugging
+- **Scalable Configuration**: Easy-to-modify sentiment categories and intents via JSON config
+
+## Project Structure
+
+```
+musora-sentiment-analysis/
+├── agents/
+│   ├── __init__.py
+│   ├── base_agent.py                    # Base class for all agents
+│   ├── language_detection_agent.py      # Language detection agent
+│   ├── translation_agent.py             # Translation agent
+│   └── sentiment_analysis_agent.py      # Sentiment analysis agent (parent context support)
+├── workflow/
+│   ├── __init__.py
+│   └── comment_processor.py             # LangGraph workflow orchestrator
+├── sql/
+│   ├── fetch_comments.sql               # Query for social media comments (with parent join)
+│   ├── fetch_musora_comments.sql        # Query for Musora internal comments (with parent join)
+│   ├── create_ml_features_table.sql     # Schema for social media table (with parent fields)
+│   ├── init_musora_table.sql            # Initialize empty Musora table (run first!)
+│   └── create_musora_ml_features_table.sql  # Full Musora schema with views (optional)
+├── config_files/
+│   ├── data_sources_config.json         # Data source configuration (NEW)
+│   ├── sentiment_config.json            # Configuration for agents and workflow
+│   └── sentiment_analysis_config.json   # Sentiment categories and intents
+├── logs/                                 # Processing logs (auto-created)
+├── LLM.py                               # LLM utility class
+├── SnowFlakeConnection.py               # Snowflake connection handler
+├── main.py                              # Main execution script (multi-source support)
+├── requirements.txt                      # Python dependencies
+├── .env                                 # Environment variables (not in git)
+├── README.md                            # This file
+└── CLAUDE.md                            # Detailed technical documentation
+```
+
+## Setup
+
+### 1. Install Dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+### 2. Configure Environment Variables
+
+Ensure your `.env` file contains the required credentials:
+
+```env
+# Snowflake
+SNOWFLAKE_USER=your_user
+SNOWFLAKE_PASSWORD=your_password
+SNOWFLAKE_ACCOUNT=your_account
+SNOWFLAKE_ROLE=your_role
+SNOWFLAKE_DATABASE=SOCIAL_MEDIA_DB
+SNOWFLAKE_WAREHOUSE=your_warehouse
+SNOWFLAKE_SCHEMA=ML_FEATURES
+
+# OpenAI
+OPENAI_API_KEY=your_openai_key
+```
+
+### 3. Create Snowflake Tables
+
+Run the SQL scripts to create the output tables:
+
+```bash
+# Execute the SQL files in Snowflake
+# For social media comments (if not already exists)
+sql/create_ml_features_table.sql
+
+# For Musora internal comments - INITIAL SETUP (First time only)
+# This creates the empty table structure
+sql/init_musora_table.sql
+```
+
+**Note**: Run `init_musora_table.sql` before the first Musora comments processing run. After that, you can optionally run `create_musora_ml_features_table.sql` to create the additional views if needed.
+
+## Usage
+
+### Basic Usage (Process All Data Sources)
+
+Process unprocessed comments from all enabled data sources:
+
+```bash
+python main.py
+```
+
+This will:
+- Process all enabled data sources (social media and Musora comments)
+- Fetch only comments that haven't been processed yet
+- Process them through the workflow using parallel workers (CPU count - 2, max 5)
+- Each batch processes and stores to Snowflake independently
+- Append new results to the existing tables (no overwrite)
+
+### Process Specific Data Source
+
+Process only social media comments:
+
+```bash
+python main.py --data-source social_media
+```
+
+Process only Musora internal comments:
+
+```bash
+python main.py --data-source musora_comments
+```
+
+### Process Limited Number of Comments
+
+Limit applies per data source:
+
+```bash
+# Process 100 comments from each enabled data source
+python main.py --limit 100
+
+# Process 100 comments from only Musora source
+python main.py --limit 100 --data-source musora_comments
+```
+
+### Sequential Processing (Debug Mode)
+
+For debugging purposes, use sequential processing:
+
+```bash
+python main.py --limit 100 --sequential
+```
+
+This processes all comments in a single batch, making it easier to debug issues.
+
+### First Run for New Data Source
+
+For the first run of Musora comments:
+
+1. **First**: Run the initialization SQL script in Snowflake:
+   ```sql
+   -- Execute in Snowflake
+   sql/init_musora_table.sql
+   ```
+
+2. **Then**: Run the processing with overwrite flag:
+   ```bash
+   python main.py --overwrite --data-source musora_comments --limit 100
+   ```
+
+**Why two steps?**
+- The fetch query checks for already-processed comments by querying the output table
+- On first run, that table doesn't exist, causing an error
+- The init script creates the empty table structure first
+- Then processing can run normally
+
+**Warning**: Overwrite will replace all existing data in the output table. Only use for initial table creation or when reprocessing from scratch.
+
+### Custom Configuration File
+
+```bash
+python main.py --config path/to/custom_config.json
+```
+
+### Command-Line Arguments
+
+- `--limit N`: Process only N comments per data source (default: 10000)
+- `--overwrite`: Overwrite existing Snowflake table (default: append mode)
+- `--config PATH`: Custom configuration file path
+- `--sequential`: Use sequential processing instead of parallel (for debugging)
+- `--data-source SOURCE`: Process only specific data source (e.g., social_media, musora_comments)
+
+### Parallel Processing
+
+The system uses multiprocessing to process comments in parallel:
+
+**Worker Calculation**:
+- Number of workers: `CPU count - 2` (max 5 workers)
+- Leaves CPU cores available for system operations
+- Example: 8-core system → 5 workers (capped at max)
+
+**Dynamic Batch Sizing**:
+- Batch size calculated as: `total_comments / num_workers`
+- Minimum batch size: 20 comments
+- Maximum batch size: 1000 comments
+- Batches ≤ 20 comments are not split
+
+**Independent Execution**:
+- Each batch runs in a separate process
+- Batches store to Snowflake immediately upon completion
+- No waiting for all batches to complete
+- Failed batches don't affect successful ones
+
+**Performance**:
+- Expected speedup: ~1.8-4.5x depending on number of workers
+- Real-time progress reporting as batches complete
+- Processing time and average per comment displayed in summary
+
+### Incremental Processing
+
+The pipeline is designed for incremental processing:
+- **Automatic deduplication**: SQL query excludes comments already in `COMMENT_SENTIMENT_FEATURES`
+- **Append-only by default**: New results are added without overwriting existing data
+- **Failed comment retry**: Comments with `success=False` are not stored and will be retried in future runs
+- **Run regularly**: Safe to run daily/weekly to process new comments
+
+## Configuration
+
+### Data Sources Configuration
+
+The `config_files/data_sources_config.json` file defines available data sources:
+
+```json
+{
+    "data_sources": {
+        "social_media": {
+            "name": "Social Media Comments",
+            "enabled": true,
+            "sql_query_file": "sql/fetch_comments.sql",
+            "output_config": {
+                "table_name": "COMMENT_SENTIMENT_FEATURES",
+                "database": "SOCIAL_MEDIA_DB",
+                "schema": "ML_FEATURES"
+            }
+        },
+        "musora_comments": {
+            "name": "Musora Internal Comments",
+            "enabled": true,
+            "sql_query_file": "sql/fetch_musora_comments.sql",
+            "output_config": {
+                "table_name": "MUSORA_COMMENT_SENTIMENT_FEATURES",
+                "database": "SOCIAL_MEDIA_DB",
+                "schema": "ML_FEATURES"
+            },
+            "additional_fields": [
+                "PERMALINK_URL",
+                "THUMBNAIL_URL"
+            ]
+        }
+    }
+}
+```
+
+**To add a new data source**: Simply add a new entry to this config file and create the corresponding SQL query file.
+
+### Agent Configuration
+
+The `config_files/sentiment_config.json` file controls agent behavior:
+
+```json
+{
+    "agents": {
+        "language_detection": {
+            "model": "gpt-5-nano",
+            "temperature": 0.0,
+            "max_retries": 3
+        },
+        "translation": {
+            "model": "gpt-5-nano",
+            "temperature": 0.3,
+            "max_retries": 3
+        },
+        "sentiment_analysis": {
+            "model": "gpt-5-nano",
+            "temperature": 0.2,
+            "max_retries": 3
+        }
+    },
+    "workflow": {
+        "description": "Batch size is calculated dynamically based on number of workers (min: 20, max: 1000)",
+        "parallel_processing": {
+            "enabled": true,
+            "worker_calculation": "CPU count - 2, max 5 workers",
+            "min_batch_size": 20,
+            "max_batch_size": 1000
+        }
+    },
+    "snowflake": {
+        "output_table": "COMMENT_SENTIMENT_FEATURES",
+        "database": "SOCIAL_MEDIA_DB",
+        "schema": "ML_FEATURES"
+    }
+}
+```
+
+**Note**: Batch size is now calculated dynamically and no longer needs to be configured manually.
+
+### Sentiment Categories Configuration
+
+The `config_files/sentiment_analysis_config.json` file defines sentiment categories and intents (easily extensible):
+
+```json
+{
+    "sentiment_polarity": {
+        "categories": [
+            {"value": "very_positive", "label": "Very Positive", "description": "..."},
+            {"value": "positive", "label": "Positive", "description": "..."},
+            {"value": "neutral", "label": "Neutral", "description": "..."},
+            {"value": "negative", "label": "Negative", "description": "..."},
+            {"value": "very_negative", "label": "Very Negative", "description": "..."}
+        ]
+    },
+    "intent": {
+        "categories": [
+            {"value": "praise", "label": "Praise", "description": "..."},
+            {"value": "question", "label": "Question", "description": "..."},
+            {"value": "request", "label": "Request", "description": "..."},
+            {"value": "feedback_negative", "label": "Negative Feedback", "description": "..."},
+            {"value": "suggestion", "label": "Suggestion", "description": "..."},
+            {"value": "humor_sarcasm", "label": "Humor/Sarcasm", "description": "..."},
+            {"value": "off_topic", "label": "Off Topic", "description": "..."},
+            {"value": "spam_selfpromo", "label": "Spam/Self-Promotion", "description": "..."}
+        ]
+    },
+    "reply_policy": {
+        "requires_reply_intents": ["question", "request"],
+        "description": "Comments with these intents should be flagged for reply"
+    },
+    "intent_settings": {
+        "multi_label": true,
+        "description": "Intent can have multiple labels as a comment can express multiple intents"
+    }
+}
+```
+
+## Adding New Agents
+
+The system is designed for easy extensibility. To add a new agent:
+
+### 1. Create Agent Class
+
+```python
+from agents.base_agent import BaseAgent
+from typing import Dict, Any
+
+class MyNewAgent(BaseAgent):
+    def __init__(self, config: Dict[str, Any], api_key: str):
+        super().__init__("MyNewAgent", config)
+        # Initialize your agent-specific components
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        # Validate input data
+        return True
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        # Implement your agent logic
+        return {"success": True, "result": "..."}
+```
+
+### 2. Update Workflow
+
+Add the agent to `workflow/comment_processor.py`:
+
+```python
+# Add to CommentState TypedDict
+new_agent_result: str
+
+# Add node
+workflow.add_node("my_new_agent", self._my_new_agent_node)
+
+# Add edges
+workflow.add_edge("translation", "my_new_agent")
+workflow.add_edge("my_new_agent", END)
+```
+
+### 3. Update Configuration
+
+Add agent config to `sentiment_config.json`:
+
+```json
+{
+    "agents": {
+        "my_new_agent": {
+            "name": "MyNewAgent",
+            "model": "gpt-4o-mini",
+            "temperature": 0.5,
+            "max_retries": 3
+        }
+    }
+}
+```
+
+## Output Schema
+
+### Social Media Comments Table
+Stored in `SOCIAL_MEDIA_DB.ML_FEATURES.COMMENT_SENTIMENT_FEATURES`
+
+### Musora Comments Table
+Stored in `SOCIAL_MEDIA_DB.ML_FEATURES.MUSORA_COMMENT_SENTIMENT_FEATURES`
+
+### Common Columns (Both Tables)
+
+| Column | Type | Description |
+|--------|------|-------------|
+| COMMENT_SK | NUMBER | Surrogate key from source |
+| COMMENT_ID | VARCHAR | Platform comment ID |
+| ORIGINAL_TEXT | VARCHAR | Original comment text |
+| **PARENT_COMMENT_ID** | **VARCHAR** | **ID of parent comment if this is a reply** |
+| **PARENT_COMMENT_TEXT** | **VARCHAR** | **Text of parent comment for context** |
+| DETECTED_LANGUAGE | VARCHAR | Detected language name |
+| LANGUAGE_CODE | VARCHAR | ISO 639-1 code |
+| IS_ENGLISH | BOOLEAN | Is comment in English |
+| TRANSLATED_TEXT | VARCHAR | English translation |
+| TRANSLATION_PERFORMED | BOOLEAN | Was translation performed |
+| SENTIMENT_POLARITY | VARCHAR | Sentiment (very_positive, positive, neutral, negative, very_negative) |
+| INTENT | VARCHAR | Multi-label intents (comma-separated) |
+| REQUIRES_REPLY | BOOLEAN | Does comment need a response |
+| SENTIMENT_CONFIDENCE | VARCHAR | Analysis confidence (high, medium, low) |
+| PROCESSING_SUCCESS | BOOLEAN | Processing status |
+| PROCESSED_AT | TIMESTAMP | Processing timestamp |
+
+### Musora-Specific Additional Columns
+
+| Column | Type | Description |
+|--------|------|-------------|
+| PERMALINK_URL | VARCHAR | Web URL path of the content |
+| THUMBNAIL_URL | VARCHAR | Thumbnail URL of the content |
+
+### Available Views
+
+**Social Media:**
+- `VW_COMMENTS_REQUIRING_REPLY`: Comments that need responses (includes parent comment info)
+- `VW_SENTIMENT_DISTRIBUTION`: Sentiment and intent statistics by channel (includes reply comment count)
+- `VW_NON_ENGLISH_COMMENTS`: Filtered view of non-English comments
+
+**Musora:**
+- `VW_MUSORA_COMMENTS_REQUIRING_REPLY`: Musora comments needing responses
+- `VW_MUSORA_SENTIMENT_DISTRIBUTION`: Musora sentiment and intent statistics
+- `VW_MUSORA_NON_ENGLISH_COMMENTS`: Non-English Musora comments
+
+## Workflow Architecture
+
+The system uses LangGraph to create a flexible, state-based workflow:
+
+```
+┌─────────────────────┐
+│  Fetch Comments     │
+│  from Snowflake     │
+│  (Unprocessed Only) │
+└──────────┬──────────┘
+           │
+           ▼
+┌─────────────────────┐
+│  Language Detection │
+│  Agent              │
+└──────────┬──────────┘
+           │
+           ▼
+      ┌────┴────┐
+      │ English?│
+      └────┬────┘
+           │
+     ┌─────┴─────┐
+     │           │
+   Yes          No
+     │           │
+     │           ▼
+     │    ┌─────────────┐
+     │    │ Translation │
+     │    │ Agent       │
+     │    └──────┬──────┘
+     │           │
+     └─────┬─────┘
+           │
+           ▼
+    ┌──────────────────┐
+    │ Sentiment        │
+    │ Analysis Agent   │
+    └─────────┬────────┘
+              │
+              ▼
+       ┌──────────────┐
+       │Store Results │
+       │to Snowflake  │
+       │(Append Mode) │
+       └──────────────┘
+```
+
+**Note**: The fetch step automatically excludes comments already present in `COMMENT_SENTIMENT_FEATURES`, enabling incremental processing.
+
+## Logging
+
+Logs are automatically created in the `logs/` directory with timestamps:
+
+```
+logs/comment_processing_20251001_143022.log
+```
+
+## Adding New Data Sources
+
+The system is designed to make adding new data sources easy:
+
+### Steps to Add a New Source:
+
+1. **Update Configuration** (`config_files/data_sources_config.json`):
+   ```json
+   "your_new_source": {
+       "name": "Your New Source Name",
+       "enabled": true,
+       "sql_query_file": "sql/fetch_your_source.sql",
+       "output_config": {
+           "table_name": "YOUR_SOURCE_SENTIMENT_FEATURES",
+           "database": "SOCIAL_MEDIA_DB",
+           "schema": "ML_FEATURES"
+       },
+       "additional_fields": ["FIELD1", "FIELD2"]  // Optional
+   }
+   ```
+
+2. **Create SQL Query File** (`sql/fetch_your_source.sql`):
+   - Fetch comments with consistent column names
+   - Include self-join for parent comments if available
+   - Exclude already-processed comments (LEFT JOIN with output table)
+
+3. **Create Table Initialization Script** (`sql/init_your_source_table.sql`):
+   - Creates empty table structure
+   - Base schema on `init_musora_table.sql`
+   - Add source-specific fields as needed
+   - **Run this in Snowflake FIRST before processing**
+
+4. **Create Full Schema** (optional):
+   - Base schema on `create_musora_ml_features_table.sql`
+   - Include views and indexes
+
+5. **Run First Time**:
+   ```bash
+   # Step 1: Run init script in Snowflake
+   sql/init_your_source_table.sql
+
+   # Step 2: Process first batch
+   python main.py --overwrite --data-source your_new_source --limit 100
+   ```
+
+**No code changes required!**
+
+## Best Practices
+
+1. **Testing**: Always test with `--limit` flag first (e.g., `--limit 100`)
+2. **New Data Sources**: Test new sources with `--sequential --limit 100` first
+3. **Debugging**: Use `--sequential` flag for easier debugging of processing issues
+4. **Incremental Processing**: Run regularly without `--overwrite` to process only new comments
+5. **Monitoring**: Check logs for processing errors and batch completion
+6. **Performance**: Use default parallel mode for production workloads
+7. **Extensibility**: Follow the base agent pattern for consistency
+8. **Error Handling**: All agents include robust error handling
+9. **Failed Comments**: Review logs for failed comments - they'll be automatically retried in future runs
+10. **Resource Management**: System automatically adapts to available CPU resources
+11. **Parent Comments**: Ensure SQL queries include parent comment joins for best accuracy
+
+## Sentiment Analysis Features
+
+### Multi-Label Intent Classification
+
+The system supports **multi-label intent classification**, meaning a single comment can have multiple intents:
+
+- **Example**: "This is amazing! What scale are you using?" → `["praise", "question"]`
+- **Example**: "Love this but can you make a tutorial on it?" → `["praise", "request"]`
+
+### Context-Aware Analysis with Parent Comment Support
+
+The sentiment analysis agent provides rich context understanding:
+
+1. **Content Context**: Uses the `content_description` field to understand what the comment is about
+2. **Parent Comment Context** (NEW): When analyzing reply comments, the system:
+   - Automatically detects when a comment is a reply
+   - Fetches the parent comment text from the database
+   - Includes parent comment in the LLM prompt
+   - Explicitly instructs the LLM that this is a reply comment
+   - Results in more accurate sentiment and intent classification
+
+**Example**:
+- Parent Comment: "Does anyone know how to play this riff?"
+- Reply Comment: "Yes!"
+- Without parent context: Might be classified as unclear/off-topic
+- With parent context: Correctly classified as answering a question
+
+This dramatically improves accuracy for:
+- Short reply comments ("Yes", "Thanks!", "Agreed")
+- Sarcastic replies (context crucial for understanding)
+- Continuation of discussions
+- Agreement/disagreement comments
+
+### Failure Handling & Reprocessing
+
+Comments that fail sentiment analysis (missing critical fields like sentiment_polarity or intents) are:
+- Marked as `success=False` in the workflow
+- **NOT stored in Snowflake**
+- **Automatically available for reprocessing** in future runs
+
+This ensures only successfully processed comments are stored, while failed comments remain available for retry.
+
+### Incremental Processing & Deduplication
+
+The pipeline automatically handles incremental processing:
+- **SQL-level deduplication**: Query excludes comments already in `COMMENT_SENTIMENT_FEATURES` using `LEFT JOIN`
+- **Automatic retry**: Failed comments (not stored) are automatically retried on next run
+- **Append-only mode**: Default behavior appends new records without overwriting
+- **Production-ready**: Safe to run daily/weekly/monthly to process new comments
+
+### Scalable Configuration
+
+To add or modify sentiment categories or intents:
+
+1. Edit `config_files/sentiment_analysis_config.json`
+2. Add/modify categories in the `sentiment_polarity` or `intent` sections
+3. Update `reply_policy.requires_reply_intents` if needed
+4. No code changes required!
+
+## Future Extensions
+
+The modular architecture supports easy addition of:
+
+- Topic classification agent
+- Entity extraction agent
+- Engagement score prediction agent
+- Named entity recognition agent
+
+Simply create a new agent inheriting from `BaseAgent` and add it to the workflow graph.
+
+## Troubleshooting
+
+### Issue: "Object does not exist or not authorized" on First Run
+
+**Error**: `Object 'SOCIAL_MEDIA_DB.ML_FEATURES.MUSORA_COMMENT_SENTIMENT_FEATURES' does not exist or not authorized`
+
+**Cause**: The fetch query tries to check for already-processed comments, but the output table doesn't exist yet on first run.
+
+**Solution**:
+1. Run the initialization script first:
+   ```sql
+   -- Execute in Snowflake
+   sql/init_musora_table.sql
+   ```
+2. Then run the processing:
+   ```bash
+   python main.py --overwrite --data-source musora_comments --limit 100
+   ```
+
+### Issue: API Rate Limits
+
+If hitting API rate limits, reduce the number of parallel workers or process fewer comments:
+```bash
+# Process fewer comments at a time
+python main.py --limit 500
+
+# Or use sequential mode
+python main.py --sequential --limit 100
+```
+
+### Issue: Memory Issues
+
+Process in smaller batches using `--limit`:
+```bash
+python main.py --limit 500
+```
+
+### Issue: Debugging Processing Errors
+
+Use sequential mode to debug issues more easily:
+```bash
+python main.py --sequential --limit 50
+```
+
+This processes all comments in a single batch with clearer error messages.
+
+### Issue: Connection Timeouts
+
+Check Snowflake credentials in `.env` and network connectivity.
+
+### Issue: Parallel Processing Not Working
+
+If multiprocessing issues occur, use sequential mode:
+```bash
+python main.py --sequential
+```
+
+## Performance
+
+### Expected Speedup
+
+Parallel processing provides significant performance improvements:
+
+- **Sequential**: 1x (baseline)
+- **2 workers**: ~1.8-1.9x faster
+- **5 workers**: ~4-4.5x faster
+
+Speedup isn't perfectly linear due to:
+- Snowflake connection overhead
+- LLM API rate limits (shared across workers)
+- I/O operations
+
+### Monitoring Performance
+
+The processing summary includes:
+- Total processing time
+- Average time per comment
+- Number of workers used
+- Batch size calculations
+- Failed batches (if any)
+
+## License
+
+Internal use only - Musora sentiment analysis project.

processing_comments/SnowFlakeConnection.py

@@ -0,0 +1,121 @@
+"""
+This class create a connection to Snowflake, run queries (read and write)
+"""
+import json
+import os
+from snowflake.snowpark import Session
+from dotenv import load_dotenv
+import logging
+logger = logging.getLogger()
+load_dotenv()
+
+class SnowFlakeConn:
+    def __init__(self):
+        self. session = self.connect_to_snowflake()
+
+
+    # =========================================================
+    def connect_to_snowflake(self):
+        # --- Snowflake connection via env vars ---
+        conn = dict(
+            user=self.get_credential("SNOWFLAKE_USER"),
+            password=self.get_credential("SNOWFLAKE_PASSWORD"),
+            account=self.get_credential("SNOWFLAKE_ACCOUNT"),
+            role=self.get_credential("SNOWFLAKE_ROLE"),
+            database=self.get_credential("SNOWFLAKE_DATABASE"),
+            warehouse=self.get_credential("SNOWFLAKE_WAREHOUSE"),
+            schema=self.get_credential("SNOWFLAKE_SCHEMA"),
+        )
+
+        session = Session.builder.configs(conn).create()
+        return session
+
+    # =========================================================
+    def get_credential(self, key):
+        return os.getenv(key)
+
+    # =========================================================
+    def run_read_query(self, query, data):
+        """
+        Executes a SQL query on Snowflake that fetch the data
+        :return: Pandas dataframe containing the query results
+        """
+
+        # Connect to Snowflake
+        try:
+            dataframe = self.session.sql(query).to_pandas()
+            dataframe.columns = dataframe.columns.str.lower()
+            print(f"reading {data} table successfully")
+            return dataframe
+        except Exception as e:
+            print(f"Error in creating/updating table: {e}")
+
+    # =========================================================
+    def store_df_to_snowflake(self, table_name, dataframe, database="SOCIAL_MEDIA_DB", schema="ML_FEATURES", overwrite=False):
+        """
+        Executes a SQL query on Snowflake that write the preprocessed data on new tables
+        :param query: SQL query string to be executed
+        :return: None
+        """
+
+        try:
+            self.session.use_database(database)
+            self.session.use_schema(schema)
+
+            dataframe = dataframe.reset_index(drop=True)
+            dataframe.columns = dataframe.columns.str.upper()
+
+            self.session.write_pandas(df=dataframe,
+                                      table_name=table_name.strip().upper(),
+                                      auto_create_table=True,
+                                      overwrite=overwrite,
+                                      use_logical_type=True)
+            print(f"Data inserted into {table_name} successfully.")
+
+        except Exception as e:
+            print(f"Error in creating/updating/inserting table: {e}")
+
+    # =========================================================
+    def execute_sql_file(self, file_path):
+        """
+        Executes SQL queries from a file
+        :param file_path: Path to SQL file
+        :return: Query result or None for DDL/DML
+        """
+        try:
+            with open(file_path, 'r', encoding='utf-8') as file:
+                sql_content = file.read()
+
+            result = self.session.sql(sql_content).collect()
+            print(f"Successfully executed SQL from {file_path}")
+            return result
+        except Exception as e:
+            print(f"Error executing SQL file {file_path}: {e}")
+            return None
+
+    # =========================================================
+    def execute_query(self, query, description="query"):
+        """
+        Executes a SQL query and returns results
+        :param query: SQL query string
+        :param description: Description of the query for logging
+        :return: Query results
+        """
+        try:
+            result = self.session.sql(query).collect()
+            print(f"Successfully executed {description}")
+            return result
+        except Exception as e:
+            print(f"Error executing {description}: {e}")
+            return None
+
+
+    # =========================================================
+    def get_data(self, data):
+        # get any sort of data based on requirement --> comments, contents, etc
+        pass
+
+    # =========================================================
+    def close_connection(self):
+        self.session.close()
+

processing_comments/agents/README.md

@@ -0,0 +1,1571 @@
+# Agents Architecture Documentation
+
+## Table of Contents
+- [Overview](#overview)
+- [Agent Architecture](#agent-architecture)
+- [Existing Agents](#existing-agents)
+- [How Agents Work](#how-agents-work)
+- [Adding New Agents](#adding-new-agents)
+- [Modifying Existing Agents](#modifying-existing-agents)
+- [Configuration System](#configuration-system)
+- [Best Practices](#best-practices)
+- [Troubleshooting](#troubleshooting)
+
+## Overview
+
+The agent system in this project is built on a modular, extensible architecture that processes social media comments through a series of specialized agents. Each agent performs a specific task (language detection, translation, sentiment analysis) and is orchestrated through a LangGraph workflow.
+
+### Key Design Principles
+
+1. **Modularity**: Each agent handles a single responsibility
+2. **Extensibility**: Easy to add new agents without modifying existing code
+3. **Consistency**: All agents inherit from a common base class
+4. **Configuration-Driven**: Agent behavior controlled through JSON config files
+5. **Error Resilience**: Robust error handling at every level
+
+### Technology Stack
+
+- **LangChain**: For LLM interactions and agent framework
+- **LangGraph**: For workflow orchestration
+- **OpenAI API**: LLM backend for NLP tasks
+- **Lingua**: Fast language detection library
+- **Python 3.x**: Core language
+
+## Agent Architecture
+
+### Directory Structure
+
+```
+agents/
+├── __init__.py                      # Module exports
+├── base_agent.py                    # Abstract base class
+├── language_detection_agent.py      # Language detection agent
+├── translation_agent.py             # Translation agent
+├── sentiment_analysis_agent.py      # Sentiment analysis agent
+└── README.md                        # This file
+```
+
+### Base Agent Class
+
+All agents inherit from `BaseAgent` (`base_agent.py`), which provides:
+
+```python
+class BaseAgent(ABC):
+    """Abstract base class for all agents"""
+
+    # Common attributes
+    - name: str                      # Agent name
+    - config: Dict[str, Any]         # Configuration dictionary
+    - model: str                     # LLM model to use
+    - temperature: float             # LLM temperature
+    - max_retries: int              # Maximum retry attempts
+
+    # Abstract methods (must be implemented)
+    @abstractmethod
+    def process(input_data: Dict) -> Dict
+    @abstractmethod
+    def validate_input(input_data: Dict) -> bool
+
+    # Common methods (inherited)
+    def get_name() -> str
+    def get_config() -> Dict
+    def log_processing(message: str, level: str)
+    def handle_error(error: Exception, context: str) -> Dict
+```
+
+### Workflow Integration
+
+Agents are orchestrated through `workflow/comment_processor.py` using LangGraph:
+
+```
+┌─────────────────────┐
+│  Language Detection │
+│  Agent              │
+└──────────┬──────────┘
+           │
+           ▼
+      ┌────┴────┐
+      │ English?│
+      └────┬────┘
+           │
+     ┌─────┴─────┐
+     │           │
+   Yes          No
+     │           │
+     │           ▼
+     │    ┌─────────────┐
+     │    │ Translation │
+     │    │ Agent       │
+     │    └──────┬──────┘
+     │           │
+     └─────┬─────┘
+           │
+           ▼
+    ┌──────────────────┐
+    │ Sentiment        │
+    │ Analysis Agent   │
+    └──────────────────┘
+```
+
+## Existing Agents
+
+### 1. Language Detection Agent
+
+**File**: `language_detection_agent.py`
+
+**Purpose**: Detects the language of comment text using a hybrid approach.
+
+**Strategy**:
+- Uses **Lingua library** for fast English detection
+- Falls back to **LLM** for non-English languages (higher accuracy)
+- Returns language name, ISO code, and confidence level
+
+**Key Methods**:
+```python
+def detect_with_lingua(text: str) -> tuple[str, str, bool]
+    # Fast detection using lingua library
+    # Returns: (language_code, language_name, is_english)
+
+def detect_with_llm(text: str) -> Dict[str, Any]
+    # LLM-based detection for nuanced analysis
+    # Returns: {language, language_code, confidence, has_text}
+
+def process(input_data: Dict) -> Dict
+    # Main processing: lingua first, LLM if not English
+```
+
+**Configuration** (`sentiment_config.json`):
+```json
+{
+    "language_detection": {
+        "model": "gpt-5-nano",
+        "temperature": 0.0,
+        "max_retries": 3
+    }
+}
+```
+
+**Input Requirements**:
+- `comment_text`: str
+
+**Output**:
+- `language`: str (e.g., "English", "Spanish")
+- `language_code`: str (ISO 639-1, e.g., "en", "es")
+- `is_english`: bool
+- `confidence`: str ("high", "medium", "low")
+- `detection_method`: str ("lingua", "llm", "default")
+- `has_text`: bool
+
+### 2. Translation Agent
+
+**File**: `translation_agent.py`
+
+**Purpose**: Translates non-English comments to English using LLM.
+
+**Strategy**:
+- Skips translation if already English
+- Uses LLM for context-aware, high-quality translation
+- Preserves tone, intent, emojis, and special characters
+- Specialized for music/education social media content
+
+**Key Methods**:
+```python
+def translate_text(text: str, source_language: str) -> Dict
+    # LLM-based translation with context preservation
+    # Returns: {translated_text, translation_confidence, notes}
+
+def process(input_data: Dict) -> Dict
+    # Main processing: checks is_english, translates if needed
+```
+
+**Configuration**:
+```json
+{
+    "translation": {
+        "model": "gpt-5-nano",
+        "temperature": 0.3,
+        "max_retries": 3
+    }
+}
+```
+
+**Input Requirements**:
+- `comment_text`: str
+- `is_english`: bool
+- `language`: str (optional, for context)
+
+**Output**:
+- `translated_text`: str
+- `translation_performed`: bool
+- `translation_confidence`: str
+- `translation_notes`: str
+
+### 3. Sentiment Analysis Agent
+
+**File**: `sentiment_analysis_agent.py`
+
+**Purpose**: Analyzes sentiment polarity, intent, and determines if reply is needed.
+
+**Strategy**:
+- Uses content description for context
+- Supports parent comment context for reply analysis
+- Multi-label intent classification
+- Differentiates genuine vs rhetorical/sarcastic questions
+- Platform-aware analysis (YouTube, Facebook, Instagram)
+
+**Key Features**:
+- **Context-Aware**: Uses content description and parent comment
+- **Multi-Label**: Can assign multiple intents to a single comment
+- **Reply Policy**: Flags comments requiring responses
+- **Rhetorical Detection**: Identifies sarcastic/rhetorical questions
+
+**Key Methods**:
+```python
+def _build_context_string(
+    content_description: str,
+    parent_comment_text: str = None,
+    platform: str = None,
+    content_title: str = None
+) -> str
+    # Builds context for LLM prompt
+    # Handles YouTube title+description vs other platforms
+
+def analyze_sentiment(
+    comment_text: str,
+    content_description: str,
+    parent_comment_text: str = None,
+    platform: str = None,
+    content_title: str = None
+) -> Dict
+    # Performs sentiment analysis with full context
+    # Returns: {sentiment_polarity, intent, requires_reply, confidence, analysis_notes}
+
+def process(input_data: Dict) -> Dict
+    # Main processing: validates input, analyzes sentiment
+```
+
+**Configuration**:
+Uses two config files:
+
+1. **Agent Config** (`sentiment_config.json`):
+```json
+{
+    "sentiment_analysis": {
+        "model": "gpt-5-nano",
+        "temperature": 0.2,
+        "max_retries": 3
+    }
+}
+```
+
+2. **Categories Config** (`sentiment_analysis_config.json`):
+```json
+{
+    "sentiment_polarity": {
+        "categories": [
+            {"value": "very_positive", "label": "Very Positive", "description": "..."},
+            {"value": "positive", "label": "Positive", "description": "..."},
+            {"value": "neutral", "label": "Neutral", "description": "..."},
+            {"value": "negative", "label": "Negative", "description": "..."},
+            {"value": "very_negative", "label": "Very Negative", "description": "..."}
+        ]
+    },
+    "intent": {
+        "categories": [
+            {"value": "praise", "description": "..."},
+            {"value": "question", "description": "..."},
+            {"value": "request", "description": "..."},
+            {"value": "feedback_negative", "description": "..."},
+            {"value": "suggestion", "description": "..."},
+            {"value": "humor_sarcasm", "description": "..."},
+            {"value": "off_topic", "description": "..."},
+            {"value": "spam_selfpromo", "description": "..."}
+        ]
+    },
+    "reply_policy": {
+        "requires_reply_intents": ["question", "request"],
+        "not_include": ["humor_sarcasm"]
+    }
+}
+```
+
+**Input Requirements**:
+- `comment_text`: str
+- `content_description`: str
+- `parent_comment_text`: str (optional)
+- `platform`: str (optional, e.g., "youtube", "facebook")
+- `content_title`: str (optional, mainly for YouTube)
+
+**Output**:
+- `sentiment_polarity`: str (one of: very_positive, positive, neutral, negative, very_negative)
+- `intent`: str (comma-separated list, e.g., "praise, question")
+- `requires_reply`: bool
+- `sentiment_confidence`: str ("high", "medium", "low")
+- `analysis_notes`: str (1-2 sentence summary)
+- `success`: bool (False if critical fields missing)
+
+### Common Patterns Across All Agents
+
+1. **JSON Response Parsing**: All agents have `_parse_llm_json_response()` method to handle markdown-wrapped JSON
+2. **Error Handling**: All use `handle_error()` from base class
+3. **Logging**: All use `log_processing()` for consistent logging
+4. **Validation**: All implement `validate_input()` before processing
+5. **State Preservation**: All preserve original input data in output
+
+## How Agents Work
+
+### Workflow Execution Flow
+
+1. **Initialization** (`CommentProcessingWorkflow.__init__`):
+   ```python
+   # Load configurations
+   lang_detect_config = config["agents"]["language_detection"]
+   translation_config = config["agents"]["translation"]
+   sentiment_config = config["agents"]["sentiment_analysis"]
+
+   # Initialize agents
+   self.language_agent = LanguageDetectionAgent(lang_detect_config, api_key)
+   self.translation_agent = TranslationAgent(translation_config, api_key)
+   self.sentiment_agent = SentimentAnalysisAgent(sentiment_config, api_key, sentiment_categories)
+
+   # Build workflow graph
+   self.workflow = self._build_workflow()
+   ```
+
+2. **Workflow Graph** (`_build_workflow()`):
+   ```python
+   workflow = StateGraph(CommentState)
+
+   # Add nodes (agents)
+   workflow.add_node("language_detection", self._language_detection_node)
+   workflow.add_node("translation", self._translation_node)
+   workflow.add_node("sentiment_analysis", self._sentiment_analysis_node)
+
+   # Define edges (control flow)
+   workflow.set_entry_point("language_detection")
+   workflow.add_conditional_edges(
+       "language_detection",
+       self._should_translate,
+       {"translate": "translation", "skip_translation": "sentiment_analysis"}
+   )
+   workflow.add_edge("translation", "sentiment_analysis")
+   workflow.add_edge("sentiment_analysis", END)
+
+   return workflow.compile()
+   ```
+
+3. **Node Execution** (Example: `_language_detection_node`):
+   ```python
+   def _language_detection_node(self, state: CommentState) -> CommentState:
+       try:
+           # Prepare input
+           input_data = {"comment_text": state["comment_text"]}
+
+           # Process with agent
+           result = self.language_agent.process(input_data)
+
+           # Update state
+           if result.get("success", False):
+               state["language"] = result.get("language", "English")
+               state["language_code"] = result.get("language_code", "en")
+               state["is_english"] = result.get("is_english", True)
+               # ... more fields
+           else:
+               # Handle error, set defaults
+               state["processing_errors"].append(result.get("error"))
+
+           return state
+       except Exception as e:
+           # Error handling
+           state["processing_errors"].append(str(e))
+           return state
+   ```
+
+4. **Decision Points** (Example: `_should_translate`):
+   ```python
+   def _should_translate(self, state: CommentState) -> str:
+       if state.get("is_english", True) or not state.get("has_text", True):
+           # Set defaults for skipped translation
+           state["translated_text"] = state["comment_text"]
+           state["translation_performed"] = False
+           return "skip_translation"
+       else:
+           return "translate"
+   ```
+
+5. **Comment Processing** (`process_comment()`):
+   ```python
+   def process_comment(self, comment_data: Dict) -> Dict:
+       # Initialize state
+       initial_state = {
+           "comment_sk": comment_data.get("comment_sk"),
+           "comment_text": comment_data.get("comment_text"),
+           # ... all fields
+           "processing_errors": [],
+           "success": True
+       }
+
+       # Run workflow
+       final_state = self.workflow.invoke(initial_state)
+
+       # Merge and return
+       return dict(final_state)
+   ```
+
+### State Management
+
+The workflow uses a `CommentState` TypedDict to pass data between agents:
+
+```python
+class CommentState(TypedDict):
+    # Input fields
+    comment_sk: int
+    comment_id: str
+    comment_text: str
+    # ... more fields
+
+    # Processing fields (populated by agents)
+    language: str
+    language_code: str
+    is_english: bool
+    translated_text: str
+    sentiment_polarity: str
+    intent: str
+    # ... more fields
+
+    # Metadata
+    processing_errors: Annotated[List[str], operator.add]
+    success: bool
+```
+
+### Error Handling Strategy
+
+1. **Agent Level**: Each agent returns `{"success": False, "error": "..."}` on failure
+2. **Node Level**: Nodes catch exceptions, set defaults, append to `processing_errors`
+3. **Workflow Level**: Workflow continues even if an agent fails (graceful degradation)
+4. **Critical Failures**: Sentiment agent marks `success=False` if critical fields missing (comment not stored)
+
+## Adding New Agents
+
+### Step-by-Step Guide
+
+#### Step 1: Create the Agent Class
+
+Create a new file in the `agents/` directory (e.g., `topic_classification_agent.py`):
+
+```python
+"""
+Topic Classification Agent
+Extracts topics and themes from comments
+"""
+
+from typing import Dict, Any
+import json
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+from agents.base_agent import BaseAgent
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class TopicClassificationAgent(BaseAgent):
+    """
+    Agent that classifies comments into predefined topics/themes.
+    """
+
+    def __init__(self, config: Dict[str, Any], api_key: str, topic_categories: Dict[str, Any]):
+        """
+        Initialize the Topic Classification Agent.
+
+        Args:
+            config: Configuration dictionary
+            api_key: OpenAI API key
+            topic_categories: Dictionary with topic categories
+        """
+        super().__init__("TopicClassificationAgent", config)
+        self.api_key = api_key
+        self.topic_categories = topic_categories
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key
+        )
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate that input contains required fields.
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        required_fields = ["comment_text"]
+        return all(field in input_data for field in required_fields)
+
+    def classify_topics(self, comment_text: str) -> Dict[str, Any]:
+        """
+        Classify comment into topics using LLM.
+
+        Args:
+            comment_text: The comment text to analyze
+
+        Returns:
+            Dictionary with topic classification results
+        """
+        # Build topic options from config
+        topic_options = "\n".join([
+            f"- {cat['value']}: {cat['description']}"
+            for cat in self.topic_categories["topics"]["categories"]
+        ])
+
+        system_prompt = f"""You are an expert at classifying music-related comments into topics.
+
+Available Topics:
+{topic_options}
+
+Return your response in JSON format with the following fields:
+- topics: array of topic values (multi-label, can have multiple topics)
+- confidence: your confidence level (high, medium, low)
+- reasoning: brief explanation of your classification
+"""
+
+        user_prompt = f"""Classify this comment into relevant topics:
+
+Comment: "{comment_text}"
+
+Return JSON only."""
+
+        try:
+            messages = [
+                SystemMessage(content=system_prompt),
+                HumanMessage(content=user_prompt)
+            ]
+
+            response = self.llm.invoke(messages)
+            result = self._parse_llm_json_response(response.content)
+
+            topics = result.get("topics", [])
+            if isinstance(topics, str):
+                topics = [topics]
+
+            topic_str = ", ".join(topics) if topics else None
+
+            return {
+                "success": True,
+                "topics": topic_str,
+                "topic_confidence": result.get("confidence", "medium"),
+                "topic_reasoning": result.get("reasoning", "")
+            }
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"JSON decode error: {str(e)}", "warning")
+            return {
+                "success": False,
+                "error": str(e)
+            }
+        except Exception as e:
+            self.log_processing(f"Topic classification failed: {str(e)}", "error")
+            return {
+                "success": False,
+                "error": str(e)
+            }
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process comment and extract topics.
+
+        Args:
+            input_data: Dictionary containing comment data
+
+        Returns:
+            Dictionary with topic classification results
+        """
+        try:
+            # Validate input
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields"
+                }
+
+            comment_text = input_data["comment_text"]
+
+            self.log_processing(f"Classifying topics for comment", "debug")
+
+            # Perform classification
+            classification_result = self.classify_topics(comment_text)
+
+            result = {
+                "success": classification_result.get("success", False),
+                "topics": classification_result.get("topics"),
+                "topic_confidence": classification_result.get("topic_confidence"),
+                "topic_reasoning": classification_result.get("topic_reasoning", "")
+            }
+
+            if "error" in classification_result:
+                result["topic_error"] = classification_result["error"]
+
+            # Preserve all original data
+            for key, value in input_data.items():
+                if key not in result:
+                    result[key] = value
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "topic_classification")
+
+    def _parse_llm_json_response(self, response_content: str) -> Dict[str, Any]:
+        """
+        Parse LLM response that may contain JSON wrapped in markdown code blocks.
+
+        Args:
+            response_content: Raw response content from LLM
+
+        Returns:
+            Parsed JSON dictionary
+
+        Raises:
+            json.JSONDecodeError: If JSON cannot be parsed
+        """
+        content = response_content.strip()
+
+        # Check if response is wrapped in markdown code block
+        if content.startswith("```json"):
+            content = content[7:]
+            if content.endswith("```"):
+                content = content[:-3]
+            content = content.strip()
+        elif content.startswith("```"):
+            content = content[3:]
+            if content.endswith("```"):
+                content = content[:-3]
+            content = content.strip()
+
+        return json.loads(content)
+```
+
+#### Step 2: Update `__init__.py`
+
+Add your agent to `agents/__init__.py`:
+
+```python
+"""
+Agents module for the sentiment analysis workflow.
+Provides modular, extensible agents for various NLP tasks.
+"""
+
+from agents.base_agent import BaseAgent
+from agents.language_detection_agent import LanguageDetectionAgent
+from agents.translation_agent import TranslationAgent
+from agents.sentiment_analysis_agent import SentimentAnalysisAgent
+from agents.topic_classification_agent import TopicClassificationAgent  # ADD THIS
+
+__all__ = [
+    "BaseAgent",
+    "LanguageDetectionAgent",
+    "TranslationAgent",
+    "SentimentAnalysisAgent",
+    "TopicClassificationAgent"  # ADD THIS
+]
+```
+
+#### Step 3: Update Configuration Files
+
+Add agent configuration to `config_files/sentiment_config.json`:
+
+```json
+{
+    "agents": {
+        "language_detection": { ... },
+        "translation": { ... },
+        "sentiment_analysis": { ... },
+        "topic_classification": {
+            "name": "TopicClassificationAgent",
+            "model": "gpt-5-nano",
+            "temperature": 0.2,
+            "max_retries": 3,
+            "description": "Classifies comments into topic categories"
+        }
+    }
+}
+```
+
+Create topic categories config (or add to existing `sentiment_analysis_config.json`):
+
+```json
+{
+    "topics": {
+        "categories": [
+            {
+                "value": "technique",
+                "label": "Technique",
+                "description": "Playing technique, finger positioning, hand coordination"
+            },
+            {
+                "value": "theory",
+                "label": "Music Theory",
+                "description": "Scales, chords, harmony, composition theory"
+            },
+            {
+                "value": "equipment",
+                "label": "Equipment",
+                "description": "Instruments, gear, accessories, software"
+            },
+            {
+                "value": "performance",
+                "label": "Performance",
+                "description": "Stage presence, live playing, performance anxiety"
+            },
+            {
+                "value": "practice",
+                "label": "Practice",
+                "description": "Practice routines, discipline, improvement tips"
+            }
+        ]
+    }
+}
+```
+
+#### Step 4: Update Workflow State
+
+Add fields to `CommentState` in `workflow/comment_processor.py`:
+
+```python
+class CommentState(TypedDict):
+    # ... existing fields ...
+
+    # Topic classification fields
+    topics: str
+    topic_confidence: str
+    topic_reasoning: str
+```
+
+#### Step 5: Add Workflow Node
+
+Add the node method to `CommentProcessingWorkflow` class:
+
+```python
+def _topic_classification_node(self, state: CommentState) -> CommentState:
+    """
+    Node for topic classification.
+
+    Args:
+        state: Current workflow state
+
+    Returns:
+        Updated state with topic classification results
+    """
+    try:
+        # Prepare input
+        input_data = {
+            "comment_text": state.get("translated_text", state["comment_text"])
+        }
+
+        # Process with topic classification agent
+        result = self.topic_agent.process(input_data)
+
+        # Update state
+        if result.get("success", False):
+            state["topics"] = result.get("topics")
+            state["topic_confidence"] = result.get("topic_confidence")
+            state["topic_reasoning"] = result.get("topic_reasoning", "")
+        else:
+            error_msg = f"Topic classification failed: {result.get('error', 'Unknown error')}"
+            state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+            state["topics"] = None
+            state["topic_confidence"] = None
+            state["topic_reasoning"] = "Topic classification failed"
+
+        logger.debug(f"Topics: {state['topics']}")
+        return state
+
+    except Exception as e:
+        error_msg = f"Topic classification node error: {str(e)}"
+        logger.error(error_msg)
+        state["processing_errors"] = state.get("processing_errors", []) + [error_msg]
+        state["topics"] = None
+        state["topic_confidence"] = None
+        state["topic_reasoning"] = "Error during topic classification"
+        return state
+```
+
+#### Step 6: Initialize Agent in Workflow
+
+Update `__init__` method:
+
+```python
+def __init__(self, config: Dict[str, Any], api_key: str):
+    # ... existing initialization ...
+
+    # Load topic categories
+    topic_categories_path = config.get("topic_categories_config", "config_files/topic_categories.json")
+    with open(topic_categories_path, 'r') as f:
+        topic_categories = json.load(f)
+
+    # Initialize topic agent
+    topic_config = config["agents"]["topic_classification"]
+    self.topic_agent = TopicClassificationAgent(topic_config, api_key, topic_categories)
+```
+
+#### Step 7: Update Workflow Graph
+
+Modify `_build_workflow()`:
+
+```python
+def _build_workflow(self) -> StateGraph:
+    workflow = StateGraph(CommentState)
+
+    # Add nodes
+    workflow.add_node("language_detection", self._language_detection_node)
+    workflow.add_node("translation", self._translation_node)
+    workflow.add_node("sentiment_analysis", self._sentiment_analysis_node)
+    workflow.add_node("topic_classification", self._topic_classification_node)  # ADD THIS
+
+    # Define edges
+    workflow.set_entry_point("language_detection")
+    workflow.add_conditional_edges(
+        "language_detection",
+        self._should_translate,
+        {"translate": "translation", "skip_translation": "sentiment_analysis"}
+    )
+    workflow.add_edge("translation", "sentiment_analysis")
+    workflow.add_edge("sentiment_analysis", "topic_classification")  # ADD THIS
+    workflow.add_edge("topic_classification", END)  # MODIFY THIS
+
+    return workflow.compile()
+```
+
+#### Step 8: Update Database Schema
+
+Add columns to your Snowflake table:
+
+```sql
+ALTER TABLE COMMENT_SENTIMENT_FEATURES
+ADD COLUMN TOPICS VARCHAR(500),
+ADD COLUMN TOPIC_CONFIDENCE VARCHAR(20),
+ADD COLUMN TOPIC_REASONING VARCHAR(1000);
+```
+
+#### Step 9: Test Your Agent
+
+Test with a small batch first:
+
+```bash
+python main.py --limit 10 --sequential
+```
+
+Check logs for any errors and verify output in Snowflake.
+
+### Quick Checklist for Adding New Agents
+
+- [ ] Create agent class inheriting from `BaseAgent`
+- [ ] Implement `validate_input()` method
+- [ ] Implement `process()` method
+- [ ] Implement `_parse_llm_json_response()` if using LLM
+- [ ] Add agent to `agents/__init__.py`
+- [ ] Add configuration to `sentiment_config.json`
+- [ ] Create/update category config file if needed
+- [ ] Add fields to `CommentState` TypedDict
+- [ ] Create node method in `CommentProcessingWorkflow`
+- [ ] Initialize agent in `__init__`
+- [ ] Add node to workflow graph
+- [ ] Update edges in workflow
+- [ ] Update database schema
+- [ ] Test with small batch
+
+## Modifying Existing Agents
+
+### Common Modifications
+
+#### 1. Change LLM Model
+
+Update `config_files/sentiment_config.json`:
+
+```json
+{
+    "agents": {
+        "sentiment_analysis": {
+            "model": "gpt-4o",  // Change from gpt-5-nano
+            "temperature": 0.2,
+            "max_retries": 3
+        }
+    }
+}
+```
+
+No code changes needed! Configuration is loaded dynamically.
+
+#### 2. Add New Sentiment Category
+
+Update `config_files/sentiment_analysis_config.json`:
+
+```json
+{
+    "sentiment_polarity": {
+        "categories": [
+            // ... existing categories ...
+            {
+                "value": "mixed",
+                "label": "Mixed",
+                "description": "Contains both positive and negative elements"
+            }
+        ]
+    }
+}
+```
+
+The agent will automatically include this in prompts. No code changes needed.
+
+#### 3. Add New Intent Category
+
+Update `config_files/sentiment_analysis_config.json`:
+
+```json
+{
+    "intent": {
+        "categories": [
+            // ... existing categories ...
+            {
+                "value": "collaboration",
+                "label": "Collaboration",
+                "description": "Seeking or offering collaboration opportunities"
+            }
+        ]
+    }
+}
+```
+
+#### 4. Modify Reply Policy
+
+Update `config_files/sentiment_analysis_config.json`:
+
+```json
+{
+    "reply_policy": {
+        "requires_reply_intents": ["question", "request", "feedback_negative"],  // Added feedback_negative
+        "not_include": ["humor_sarcasm", "spam_selfpromo"]  // Added spam_selfpromo
+    }
+}
+```
+
+#### 5. Adjust Temperature for Better Results
+
+If getting inconsistent results, adjust temperature:
+
+```json
+{
+    "agents": {
+        "sentiment_analysis": {
+            "model": "gpt-5-nano",
+            "temperature": 0.1,  // Lower = more consistent, less creative
+            "max_retries": 3
+        }
+    }
+}
+```
+
+#### 6. Add Context to Sentiment Analysis
+
+Modify `_build_context_string()` in `sentiment_analysis_agent.py`:
+
+```python
+def _build_context_string(self, content_description: str, parent_comment_text: str = None,
+                          platform: str = None, content_title: str = None,
+                          channel_name: str = None) -> str:  # ADD channel_name
+    """Build context string for sentiment analysis."""
+    context_parts = []
+
+    # ... existing code ...
+
+    # ADD THIS
+    if channel_name:
+        context_parts.append(f"Channel: {channel_name}")
+
+    return "\n".join(context_parts)
+```
+
+Then update the `analyze_sentiment()` method to accept and pass `channel_name`.
+
+#### 7. Improve Language Detection Accuracy
+
+Modify `language_detection_agent.py` to add more languages to LINGUA_TO_ISO:
+
+```python
+LINGUA_TO_ISO = {
+    # ... existing mappings ...
+    Language.VIETNAMESE: "vi",
+    Language.THAI: "th",
+    Language.INDONESIAN: "id",
+    # Add more as needed
+}
+```
+
+#### 8. Customize Translation Prompt
+
+Modify `translate_text()` in `translation_agent.py`:
+
+```python
+system_prompt = """You are a professional translator specializing in social media content related to music and education.
+Translate the given text from the source language to English. The text is a comment on a musical content.
+Preserve the tone, intent, and any emojis or special characters.
+For informal social media language, maintain the casual tone in translation.
+
+// ADD THESE GUIDELINES:
+Special Instructions:
+- Preserve musical terminology (e.g., "legato", "staccato") untranslated
+- Translate instrument names (e.g., "guitarra" → "guitar")
+- Keep artist names and brand names in original language
+- Maintain slang and colloquialisms when possible
+
+Return your response in JSON format with the following fields:
+- translated_text: The English translation
+- translation_confidence: Your confidence level (high, medium, low)
+- notes: Any important notes about the translation (optional)
+"""
+```
+
+#### 9. Add Retry Logic for Failed Analyses
+
+Modify `process()` in `sentiment_analysis_agent.py`:
+
+```python
+def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+    try:
+        # ... existing validation code ...
+
+        # ADD RETRY LOGIC
+        max_attempts = self.max_retries
+        for attempt in range(max_attempts):
+            analysis_result = self.analyze_sentiment(
+                comment_text, content_description,
+                parent_comment_text, platform, content_title
+            )
+
+            if analysis_result.get("success"):
+                break
+
+            if attempt < max_attempts - 1:
+                self.log_processing(f"Attempt {attempt + 1} failed, retrying...", "warning")
+
+        # ... rest of existing code ...
+```
+
+#### 10. Add Custom Validation Rules
+
+Modify `validate_input()` in any agent:
+
+```python
+def validate_input(self, input_data: Dict[str, Any]) -> bool:
+    """Validate that input contains required fields."""
+    required_fields = ["comment_text", "content_description"]
+
+    # Check required fields exist
+    if not all(field in input_data for field in required_fields):
+        return False
+
+    # ADD CUSTOM VALIDATION
+    # Ensure comment_text is not empty or too short
+    comment_text = input_data.get("comment_text", "")
+    if not comment_text or len(comment_text.strip()) < 2:
+        self.log_processing("Comment text too short or empty", "warning")
+        return False
+
+    # Ensure content_description exists
+    content_desc = input_data.get("content_description", "")
+    if not content_desc or content_desc.strip() == "":
+        self.log_processing("Content description missing", "warning")
+        return False
+
+    return True
+```
+
+### Testing Modified Agents
+
+After making modifications, always test:
+
+```bash
+# Test with a small batch
+python main.py --limit 10 --sequential
+
+# Check specific data source
+python main.py --limit 10 --sequential --data-source social_media
+
+# Review logs for errors
+tail -f logs/comment_processing_*.log
+```
+
+## Configuration System
+
+### Configuration Files Overview
+
+```
+config_files/
+├── sentiment_config.json              # Agent behavior config
+├── sentiment_analysis_config.json     # Sentiment categories and intents
+└── data_sources_config.json           # Data source configuration
+```
+
+### Agent Configuration Structure
+
+**File**: `sentiment_config.json`
+
+```json
+{
+    "agents": {
+        "agent_name": {
+            "name": "AgentClassName",
+            "model": "gpt-5-nano",        // LLM model to use
+            "temperature": 0.0,            // Creativity (0.0 = deterministic, 1.0 = creative)
+            "max_retries": 3,              // Max retry attempts
+            "description": "What this agent does"
+        }
+    },
+    "workflow": {
+        "parallel_processing": {
+            "enabled": true,
+            "worker_calculation": "CPU count - 2, max 5 workers",
+            "min_batch_size": 20,
+            "max_batch_size": 1000
+        }
+    }
+}
+```
+
+### Temperature Guidelines
+
+- **0.0 - 0.1**: Deterministic, consistent (good for classification)
+- **0.2 - 0.4**: Slight variation, mostly consistent (good for sentiment analysis)
+- **0.5 - 0.7**: Balanced creativity and consistency (good for translation)
+- **0.8 - 1.0**: Creative, varied (good for content generation)
+
+### Model Selection Guidelines
+
+- **gpt-5-nano**: Fast, cheap, good for simple tasks
+- **gpt-4o-mini**: Balanced speed/quality, good for most tasks
+- **gpt-4o**: High quality, slower, good for complex analysis
+
+### Category Configuration Structure
+
+**File**: `sentiment_analysis_config.json`
+
+```json
+{
+    "category_type": {
+        "categories": [
+            {
+                "value": "machine_readable_value",  // Used in code/DB
+                "label": "Human Readable Label",    // Used in UI
+                "description": "Detailed description for LLM prompt"
+            }
+        ]
+    }
+}
+```
+
+### Loading Configuration in Code
+
+```python
+# In workflow/__init__ or agent __init__
+import json
+import os
+
+# Load agent config
+with open('config_files/sentiment_config.json', 'r') as f:
+    config = json.load(f)
+
+agent_config = config["agents"]["agent_name"]
+
+# Load category config
+with open('config_files/sentiment_analysis_config.json', 'r') as f:
+    categories = json.load(f)
+
+sentiment_categories = categories["sentiment_polarity"]["categories"]
+```
+
+## Best Practices
+
+### Agent Development
+
+1. **Single Responsibility**: Each agent should do one thing well
+2. **Fail Gracefully**: Always return structured error responses
+3. **Preserve Data**: Never lose original input data - pass it through
+4. **Log Everything**: Use `log_processing()` for debugging
+5. **Validate Early**: Check inputs before processing
+6. **Configuration Over Code**: Use config files for behavior changes
+7. **Test Incrementally**: Test with `--limit 10 --sequential` first
+
+### Prompt Engineering
+
+1. **Be Specific**: Clearly define expected output format
+2. **Use Examples**: Include few-shot examples in prompts
+3. **Request JSON**: Always request JSON format for structured data
+4. **Handle Edge Cases**: Document edge cases in prompts
+5. **Provide Context**: Give LLM all relevant context
+6. **Set Constraints**: Clearly define boundaries and limitations
+
+Example of good prompt structure:
+
+```python
+system_prompt = """You are an expert at [TASK].
+
+Your task is to:
+1. [Step 1]
+2. [Step 2]
+3. [Step 3]
+
+Context: [Explain the context]
+
+Rules:
+- Rule 1
+- Rule 2
+- Rule 3
+
+Examples:
+- Input: "..." → Output: {...}
+- Input: "..." → Output: {...}
+
+Return your response in JSON format with the following fields:
+- field1: description
+- field2: description
+"""
+```
+
+### Error Handling
+
+1. **Try-Catch Everything**: Wrap all processing in try-catch
+2. **Specific Error Messages**: Make errors actionable
+3. **Graceful Degradation**: Continue workflow even if one agent fails
+4. **Error Accumulation**: Collect errors in `processing_errors` list
+5. **Critical vs Non-Critical**: Distinguish between recoverable and fatal errors
+
+Example:
+
+```python
+def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+    try:
+        # Validate
+        if not self.validate_input(input_data):
+            return {
+                "success": False,
+                "error": "Invalid input: missing required fields",
+                **input_data  # Preserve original data
+            }
+
+        # Process
+        result = self.do_processing(input_data)
+
+        # Check result
+        if not result.get("success"):
+            return {
+                "success": False,
+                "error": result.get("error", "Unknown error"),
+                **input_data
+            }
+
+        # Return success
+        return {
+            "success": True,
+            "output_field": result["output"],
+            **input_data
+        }
+
+    except Exception as e:
+        return self.handle_error(e, "process")
+```
+
+### Testing
+
+1. **Unit Test Agents**: Test agents independently before integration
+2. **Small Batches**: Always test with `--limit 10` first
+3. **Sequential Mode**: Use `--sequential` for debugging
+4. **Check Logs**: Review logs after every test run
+5. **Validate Output**: Check Snowflake results
+6. **Test Edge Cases**: Empty text, emojis only, very long text, special characters
+
+Test script example:
+
+```python
+# test_agent.py
+from agents.sentiment_analysis_agent import SentimentAnalysisAgent
+import json
+
+# Load config
+with open('config_files/sentiment_config.json', 'r') as f:
+    config = json.load(f)
+with open('config_files/sentiment_analysis_config.json', 'r') as f:
+    categories = json.load(f)
+
+# Initialize agent
+agent = SentimentAnalysisAgent(
+    config["agents"]["sentiment_analysis"],
+    "your-api-key",
+    categories
+)
+
+# Test cases
+test_cases = [
+    {"comment_text": "This is amazing!", "content_description": "Guitar tutorial"},
+    {"comment_text": "😊😊😊", "content_description": "Piano cover"},
+    {"comment_text": "What scale is this?", "content_description": "Blues solo"},
+]
+
+for test in test_cases:
+    result = agent.process(test)
+    print(f"Input: {test['comment_text']}")
+    print(f"Result: {result}")
+    print("---")
+```
+
+### Performance Optimization
+
+1. **Batch Processing**: Process comments in batches (handled by workflow)
+2. **Parallel Workers**: Use multiprocessing for large batches
+3. **Minimize LLM Calls**: Cache results when possible
+4. **Optimize Prompts**: Shorter prompts = faster responses
+5. **Choose Right Model**: Use gpt-5-nano for simple tasks
+
+### Code Organization
+
+1. **One Agent Per File**: Don't combine multiple agents
+2. **Helper Methods**: Use private methods (\_method\_name) for internal logic
+3. **Type Hints**: Always use type hints for parameters and returns
+4. **Docstrings**: Document all public methods
+5. **Constants**: Define constants at class level
+
+Example structure:
+
+```python
+class MyAgent(BaseAgent):
+    # Constants
+    DEFAULT_VALUE = "default"
+    MAX_LENGTH = 1000
+
+    def __init__(self, config, api_key):
+        """Initialize agent."""
+        super().__init__("MyAgent", config)
+        # ... initialization
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """Validate input data."""
+        # ... validation
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Main processing method."""
+        # ... processing
+
+    def _helper_method(self, data: str) -> str:
+        """Private helper method."""
+        # ... helper logic
+
+    def _parse_llm_json_response(self, response: str) -> Dict[str, Any]:
+        """Parse LLM JSON response."""
+        # ... parsing
+```
+
+## Troubleshooting
+
+### Common Issues
+
+#### Issue 1: Agent Returns Empty Results
+
+**Symptoms**: Agent succeeds but returns None or empty strings for key fields
+
+**Causes**:
+- LLM not following JSON format
+- JSON parsing failing silently
+- Missing fields in LLM response
+
+**Solutions**:
+1. Check logs for JSON parsing warnings
+2. Add validation after LLM call:
+   ```python
+   result = self._parse_llm_json_response(response.content)
+
+   # Validate result
+   if not result.get("sentiment_polarity"):
+       return {
+           "success": False,
+           "error": "Missing sentiment_polarity in LLM response"
+       }
+   ```
+3. Improve prompt to be more specific about required fields
+4. Add examples to prompt showing exact JSON structure
+
+#### Issue 2: JSON Parsing Errors
+
+**Symptoms**: `JSON decode error` in logs
+
+**Causes**:
+- LLM returns markdown-wrapped JSON
+- LLM includes explanatory text before/after JSON
+- Malformed JSON from LLM
+
+**Solutions**:
+1. Use `_parse_llm_json_response()` helper (already handles markdown)
+2. Add more explicit prompt:
+   ```python
+   user_prompt = """...
+
+   Return ONLY valid JSON, no explanation or markdown. Just the raw JSON object.
+   """
+   ```
+3. Add fallback parsing:
+   ```python
+   try:
+       result = json.loads(content)
+   except json.JSONDecodeError:
+       # Try to extract JSON from text
+       import re
+       json_match = re.search(r'\{.*\}', content, re.DOTALL)
+       if json_match:
+           result = json.loads(json_match.group())
+       else:
+           raise
+   ```
+
+#### Issue 3: Inconsistent Results
+
+**Symptoms**: Same comment gets different classifications on reruns
+
+**Causes**:
+- Temperature too high
+- Prompt too vague
+- Model inconsistency
+
+**Solutions**:
+1. Lower temperature to 0.0 - 0.2 for classification tasks
+2. Make prompt more specific and rule-based
+3. Add examples to prompt
+4. Use a more consistent model (gpt-5-nano vs gpt-4o)
+
+#### Issue 4: Agent Too Slow
+
+**Symptoms**: Processing takes very long
+
+**Causes**:
+- Large LLM model
+- Complex prompts
+- Sequential processing
+- API rate limits
+
+**Solutions**:
+1. Use faster model (gpt-5-nano instead of gpt-4o)
+2. Simplify prompt (shorter = faster)
+3. Enable parallel processing (already default)
+4. Increase batch size (if not hitting rate limits)
+5. Consider caching repeated analyses
+
+#### Issue 5: Agent Failing Validation
+
+**Symptoms**: `validate_input()` returns False, agent skips processing
+
+**Causes**:
+- Missing required fields in input
+- Empty or None values
+- Wrong data types
+
+**Solutions**:
+1. Check workflow node - ensure all required fields passed:
+   ```python
+   input_data = {
+       "comment_text": state.get("translated_text", state["comment_text"]),
+       "content_description": state["content_description"],
+       # Add all required fields
+   }
+   ```
+2. Add logging to validation:
+   ```python
+   def validate_input(self, input_data: Dict[str, Any]) -> bool:
+       for field in required_fields:
+           if field not in input_data:
+               self.log_processing(f"Missing field: {field}", "error")
+               return False
+       return True
+   ```
+
+#### Issue 6: Workflow Not Running New Agent
+
+**Symptoms**: New agent not being called, no logs from new agent
+
+**Causes**:
+- Forgot to add node to workflow graph
+- Forgot to initialize agent
+- Workflow edges not connected
+
+**Solutions**:
+1. Verify agent initialized in `__init__`:
+   ```python
+   self.new_agent = NewAgent(config, api_key)
+   ```
+2. Verify node added:
+   ```python
+   workflow.add_node("new_agent", self._new_agent_node)
+   ```
+3. Verify edges:
+   ```python
+   workflow.add_edge("previous_agent", "new_agent")
+   workflow.add_edge("new_agent", END)
+   ```
+4. Check for exceptions in workflow compilation
+
+#### Issue 7: Database Insert Fails
+
+**Symptoms**: Processing succeeds but data not in Snowflake
+
+**Causes**:
+- Missing columns in database
+- Data type mismatch
+- Field name mismatch
+
+**Solutions**:
+1. Check column exists:
+   ```sql
+   DESC TABLE COMMENT_SENTIMENT_FEATURES;
+   ```
+2. Add column if missing:
+   ```sql
+   ALTER TABLE COMMENT_SENTIMENT_FEATURES
+   ADD COLUMN NEW_FIELD VARCHAR(500);
+   ```
+3. Check field names match exactly (case-sensitive)
+4. Check main.py result_df construction includes new fields
+
+### Debugging Tips
+
+1. **Enable Debug Logging**: Set log level to DEBUG in main.py
+2. **Print State**: Add print statements in workflow nodes to see state
+3. **Test Agent Directly**: Test agent outside workflow first
+4. **Use Sequential Mode**: `--sequential` flag for clearer debugging
+5. **Check API Logs**: Review OpenAI API dashboard for errors
+6. **Validate JSON**: Use online JSON validator for config files
+7. **Check Git Status**: Ensure all files saved and changes committed
+
+### Getting Help
+
+1. **Check Logs**: Always check `logs/` directory first
+2. **Review This README**: Answers to most questions are here
+3. **Test Incrementally**: Isolate the problem to one agent
+4. **Use Small Batches**: Test with `--limit 5` for faster iteration
+5. **Document Issues**: Keep notes on what you tried
+
+## Conclusion
+
+This agent architecture provides a flexible, maintainable foundation for processing social media comments. Key takeaways:
+
+- **Base class pattern** ensures consistency
+- **LangGraph workflow** enables flexible orchestration
+- **Configuration-driven** design minimizes code changes
+- **Error resilience** at every level
+- **Extensible by design** - easy to add new agents
+
+For questions or issues, refer to the main project README or review the existing agent implementations for patterns and examples.
+
+---
+
+**Last Updated**: 2026-01-15
+**Version**: 1.0
+**Maintainer**: Musora Development Team

processing_comments/agents/__init__.py

@@ -0,0 +1,14 @@
+"""
+Agents module for the sentiment analysis workflow.
+Provides modular, extensible agents for various NLP tasks.
+"""
+
+from agents.base_agent import BaseAgent
+from agents.language_detection_agent import LanguageDetectionAgent
+from agents.translation_agent import TranslationAgent
+
+__all__ = [
+    "BaseAgent",
+    "LanguageDetectionAgent",
+    "TranslationAgent"
+]

processing_comments/agents/base_agent.py

@@ -0,0 +1,104 @@
+"""
+Base Agent class for all agents in the workflow
+This provides a common interface and structure for extensibility
+"""
+
+from abc import ABC, abstractmethod
+from typing import Dict, Any, Optional
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAgent(ABC):
+    """
+    Abstract base class for all agents in the agentic workflow.
+    Provides common functionality and enforces consistent interface.
+    """
+
+    def __init__(self, name: str, config: Dict[str, Any]):
+        """
+        Initialize the base agent.
+
+        Args:
+            name: Name of the agent
+            config: Configuration dictionary for the agent
+        """
+        self.name = name
+        self.config = config
+        self.model = config.get("model", "gpt-4o-mini")
+        self.temperature = config.get("temperature", 0.7)
+        self.max_retries = config.get("max_retries", 3)
+        logger.info(f"Initialized {self.name} with model {self.model}")
+
+    @abstractmethod
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process input data and return results.
+        This method must be implemented by all concrete agent classes.
+
+        Args:
+            input_data: Dictionary containing input data for processing
+
+        Returns:
+            Dictionary containing processing results
+        """
+        pass
+
+    @abstractmethod
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate input data before processing.
+
+        Args:
+            input_data: Dictionary containing input data
+
+        Returns:
+            True if input is valid, False otherwise
+        """
+        pass
+
+    def get_name(self) -> str:
+        """Get the agent name."""
+        return self.name
+
+    def get_config(self) -> Dict[str, Any]:
+        """Get the agent configuration."""
+        return self.config
+
+    def log_processing(self, message: str, level: str = "info"):
+        """
+        Log processing information.
+
+        Args:
+            message: Log message
+            level: Log level (info, warning, error, debug)
+        """
+        log_method = getattr(logger, level, logger.info)
+        log_method(f"[{self.name}] {message}")
+
+    def handle_error(self, error: Exception, context: str = "") -> Dict[str, Any]:
+        """
+        Handle errors consistently across all agents.
+
+        Args:
+            error: The exception that occurred
+            context: Additional context about the error
+
+        Returns:
+            Error dictionary with details
+        """
+        error_msg = f"Error in {self.name}"
+        if context:
+            error_msg += f" ({context})"
+        error_msg += f": {str(error)}"
+
+        logger.error(error_msg)
+
+        return {
+            "success": False,
+            "error": str(error),
+            "agent": self.name,
+            "context": context
+        }

processing_comments/agents/language_detection_agent.py

@@ -0,0 +1,292 @@
+"""
+Language Detection Agent
+Detects the language of social media comments using lingua library and LLM fallback
+"""
+
+from typing import Dict, Any
+import json
+from lingua import Language, LanguageDetectorBuilder
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+from agents.base_agent import BaseAgent
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class LanguageDetectionAgent(BaseAgent):
+    """
+    Agent that detects the language of text comments.
+    Uses lingua library for fast English detection, then LLM for non-English languages.
+    """
+
+    # Lingua to ISO 639-1 language code mapping
+    LINGUA_TO_ISO = {
+        Language.ENGLISH: "en",
+        Language.SPANISH: "es",
+        Language.FRENCH: "fr",
+        Language.GERMAN: "de",
+        Language.ITALIAN: "it",
+        Language.PORTUGUESE: "pt",
+        Language.RUSSIAN: "ru",
+        Language.JAPANESE: "ja",
+        Language.KOREAN: "ko",
+        Language.CHINESE: "zh",
+        Language.ARABIC: "ar",
+        Language.HINDI: "hi",
+        Language.DUTCH: "nl",
+        Language.SWEDISH: "sv",
+        Language.POLISH: "pl",
+        Language.TURKISH: "tr"
+    }
+
+    def __init__(self, config: Dict[str, Any], api_key: str):
+        """
+        Initialize the Language Detection Agent.
+
+        Args:
+            config: Configuration dictionary
+            api_key: OpenAI API key
+        """
+        super().__init__("LanguageDetectionAgent", config)
+        self.api_key = api_key
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key
+        )
+
+        # Initialize lingua detector with all languages
+        self.detector = LanguageDetectorBuilder.from_all_languages().build()
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate that input contains required fields.
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        return "comment_text" in input_data and input_data["comment_text"]
+
+    def detect_with_lingua(self, text: str) -> tuple[str, str, bool]:
+        """
+        Detect language using lingua library.
+
+        Args:
+            text: Text to analyze
+
+        Returns:
+            Tuple of (language_code, language_name, is_english)
+        """
+        try:
+            # Clean text
+            cleaned_text = text.strip()
+            if not cleaned_text or len(cleaned_text) < 3:
+                return "en", "English", True  # Default for very short text
+
+            # Detect language with lingua
+            detected_language = self.detector.detect_language_of(cleaned_text)
+
+            if detected_language is None:
+                # If detection fails, default to English
+                return "en", "English", True
+
+            # Check if it's English
+            if detected_language == Language.ENGLISH:
+                return "en", "English", True
+
+            # Map lingua language to ISO code
+            lang_code = self.LINGUA_TO_ISO.get(detected_language, "unknown")
+            lang_name = detected_language.name.capitalize()
+
+            return lang_code, lang_name, False
+
+        except Exception as e:
+            logger.warning(f"Lingua detection failed: {str(e)}")
+            # If detection fails, default to English
+            return "en", "English", True
+
+    def _parse_llm_json_response(self, response_content: str) -> Dict[str, Any]:
+        """
+        Parse LLM response that may contain JSON wrapped in markdown code blocks.
+
+        Args:
+            response_content: Raw response content from LLM
+
+        Returns:
+            Parsed JSON dictionary
+
+        Raises:
+            json.JSONDecodeError: If JSON cannot be parsed
+        """
+        content = response_content.strip()
+
+        # Check if response is wrapped in markdown code block
+        if content.startswith("```json"):
+            # Remove ```json prefix and ``` suffix
+            content = content[7:]  # Remove ```json
+            if content.endswith("```"):
+                content = content[:-3]  # Remove trailing ```
+            content = content.strip()
+        elif content.startswith("```"):
+            # Remove generic ``` code block
+            content = content[3:]
+            if content.endswith("```"):
+                content = content[:-3]
+            content = content.strip()
+
+        # Parse the cleaned JSON
+        return json.loads(content)
+
+    def detect_with_llm(self, text: str) -> Dict[str, Any]:
+        """
+        Detect language using LLM for more nuanced detection.
+
+        Args:
+            text: Text to analyze
+
+        Returns:
+            Dictionary with language detection results
+        """
+        system_prompt = """You are a language detection expert. Analyze the given text and detect its language.
+For text with only emojis, special characters, or minimal content, classify as "English". Comment is about a music content, so having links or using musician name is normal and still be english. 
+Return your response in JSON format with the following fields:
+- language: The detected language name (e.g., "English", "Spanish", "French")
+- language_code: ISO 639-1 language code (e.g., "en", "es", "fr")
+- confidence: Your confidence level (high, medium, low)
+- has_text: boolean indicating if there is actual textual content (not just emojis/symbols)
+"""
+
+        user_prompt = f"""Detect the language of this comment related to a musical content:
+
+"{text}"
+
+Return JSON only."""
+
+        try:
+            messages = [
+                SystemMessage(content=system_prompt),
+                HumanMessage(content=user_prompt)
+            ]
+
+            response = self.llm.invoke(messages)
+
+            # Parse the response using helper function
+            result = self._parse_llm_json_response(response.content)
+
+            # If no text content, default to English
+            if not result.get("has_text", True):
+                result["language"] = "English"
+                result["language_code"] = "en"
+
+            return result
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"LLM response JSON parsing failed: {str(e)}", "warning")
+            self.log_processing(f"Raw response: {response.content[:200]}", "debug")
+            return {
+                "language": "English",
+                "language_code": "en",
+                "confidence": "low",
+                "has_text": True
+            }
+        except Exception as e:
+            self.log_processing(f"LLM detection failed: {str(e)}", "warning")
+            return {
+                "language": "English",
+                "language_code": "en",
+                "confidence": "low",
+                "has_text": True
+            }
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process comment and detect its language.
+        Strategy: Use lingua first. If English, done. If not English, use LLM for better accuracy.
+
+        Args:
+            input_data: Dictionary containing comment_text and other metadata
+
+        Returns:
+            Dictionary with language detection results
+        """
+        try:
+            # Validate input
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing comment_text",
+                    "language": "English",
+                    "language_code": "en",
+                    "is_english": True
+                }
+
+            comment_text = input_data["comment_text"]
+
+            # Check for empty or emoji-only content
+            if not comment_text or len(comment_text.strip()) == 0:
+                return {
+                    "success": True,
+                    "comment_text": comment_text,
+                    "language": "English",
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "high",
+                    "detection_method": "default",
+                    "has_text": False
+                }
+
+            # Step 1: Use lingua for initial detection
+            lingua_lang_code, lingua_lang_name, is_english = self.detect_with_lingua(comment_text)
+
+            # Step 2: If English, we're done (lingua is good at detecting English)
+            if is_english:
+                result = {
+                    "success": True,
+                    "comment_text": comment_text,
+                    "language": "English",
+                    "language_code": "en",
+                    "is_english": True,
+                    "confidence": "high",
+                    "detection_method": "lingua",
+                    "has_text": True
+                }
+            else:
+                # Step 3: If not English, use LLM for more accurate detection
+                llm_result = self.detect_with_llm(comment_text)
+                language = llm_result.get("language", lingua_lang_name)
+                language_code = llm_result.get("language_code", lingua_lang_code)
+                confidence = llm_result.get("confidence", "medium")
+                has_text = llm_result.get("has_text", True)
+                if language_code == "en" or language == "English":
+                    is_english=True
+
+                result = {
+                    "success": True,
+                    "comment_text": comment_text,
+                    "language": language,
+                    "language_code": language_code,
+                    "is_english": is_english,
+                    "confidence": confidence,
+                    "detection_method": "llm",
+                    "has_text": has_text
+                }
+
+            # Preserve original metadata
+            for key, value in input_data.items():
+                if key not in result:
+                    result[key] = value
+
+            self.log_processing(
+                f"Detected language: {result['language']} ({result['language_code']}) - "
+                f"Method: {result['detection_method']}",
+                "debug"
+            )
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "language detection")

processing_comments/agents/sentiment_analysis_agent.py

@@ -0,0 +1,381 @@
+"""
+Sentiment Analysis Agent
+Extracts sentiment polarity, intent, and determines if reply is needed
+"""
+
+from typing import Dict, Any, List, Optional
+import json
+import re
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+from agents.base_agent import BaseAgent
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Reply policy constants — must stay in sync with reply_policy in sentiment_analysis_config.json
+_REQUIRES_REPLY_INTENTS = {"question", "request", "subscription"}
+_NO_REPLY_INTENTS = {"humor_sarcasm"}
+
+# Compiled regexes for content description parsing (compiled once at module load)
+_RE_FOLLOW_SECTION = re.compile(r"^Follow\b", re.IGNORECASE)
+_RE_ARROW_LINK = re.compile(r"^►")
+_RE_URL_ONLY = re.compile(r"^https?://\S+$")
+_RE_TIMESTAMP = re.compile(r"^\d+:\d+\s*[-–]\s*(.*)")
+
+
+class SentimentAnalysisAgent(BaseAgent):
+    """
+    Agent that analyzes comment sentiment, intent, and reply requirements.
+
+    Design decisions:
+    - System prompt is built once at init (static across all calls)
+    - requires_reply is computed deterministically in Python, not by the LLM
+    - LLM output is validated against config-defined allowed value sets
+    - Content descriptions are parsed to strip URLs, timestamps, and social sections
+    - Parent comments are passed as read-only context; classification targets the
+      TARGET comment only
+    """
+
+    def __init__(self, config: Dict[str, Any], api_key: str, sentiment_categories: Dict[str, Any]):
+        """
+        Initialize the Sentiment Analysis Agent.
+
+        Args:
+            config: Agent configuration dictionary
+            api_key: OpenAI API key
+            sentiment_categories: Loaded sentiment_analysis_config.json dict
+        """
+        super().__init__("SentimentAnalysisAgent", config)
+        self.api_key = api_key
+        self.sentiment_categories = sentiment_categories
+
+        # Pre-compute valid value sets from config for O(1) validation
+        self._valid_polarities = {
+            cat["value"] for cat in sentiment_categories["sentiment_polarity"]["categories"]
+        }
+        self._valid_intents = {
+            cat["value"] for cat in sentiment_categories["intent"]["categories"]
+        }
+
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key,
+            model_kwargs={"response_format": {"type": "json_object"}}
+        )
+
+        # Build system prompt once at init — reused for every LLM call
+        self._system_prompt = self._build_system_prompt()
+
+    # ------------------------------------------------------------------
+    # Prompt construction
+    # ------------------------------------------------------------------
+
+    def _build_system_prompt(self) -> str:
+        """
+        Build a compact, static system prompt from the sentiment config.
+        Pulls category descriptions directly from config so changes to
+        sentiment_analysis_config.json are automatically reflected.
+        """
+        polarity_lines = "\n".join(
+            f"- {cat['value']}: {cat['description']}"
+            for cat in self.sentiment_categories["sentiment_polarity"]["categories"]
+        )
+
+        intent_lines = "\n".join(
+            f"- {cat['value']}: {cat['description']}"
+            for cat in self.sentiment_categories["intent"]["categories"]
+        )
+
+        return (
+            "Classify a social media comment about musical content.\n\n"
+            "RULE: Analyze ONLY the TARGET comment. "
+            "The parent comment is context only — do not extract sentiment or intent from it.\n\n"
+            "Return JSON only:\n"
+            '{"sentiment_polarity": <value>, "intents": [<values>], '
+            '"confidence": "high"|"medium"|"low", "analysis_notes": "<1-2 sentences>"}\n\n'
+            f"POLARITY (pick one):\n{polarity_lines}\n\n"
+            f"INTENTS (multi-label, pick all that apply):\n{intent_lines}\n\n"
+            "Rhetorical/sarcasm rules:\n"
+            "- Rhetorical questions → humor_sarcasm or feedback_negative, NOT question\n"
+            "- Sarcastic suggestions → feedback_negative, NOT suggestion\n"
+            "- Sarcastic requests → feedback_negative, NOT request\n"
+            "- Only use question/request/suggestion for GENUINE expressions"
+        )
+
+    def _build_user_prompt(
+        self,
+        comment_text: str,
+        content_description: str,
+        parent_comment_text: Optional[str] = None,
+        platform: Optional[str] = None,
+        content_title: Optional[str] = None,
+    ) -> str:
+        """
+        Build the compact user prompt with parsed, truncated context.
+
+        YouTube stores the video title separately from the description, so they
+        are combined here. Other platforms already embed the title in the
+        description, so only the parsed description is used.
+        """
+        parsed_description = self._parse_content_description(content_description)
+
+        if platform and platform.lower() == "youtube" and content_title and str(content_title).strip():
+            content_context = f"{content_title.strip()} — {parsed_description}"[:500]
+        else:
+            content_context = parsed_description
+
+        parts = [f"Content: {content_context}"]
+
+        if parent_comment_text and str(parent_comment_text).strip():
+            parent_snippet = str(parent_comment_text).strip()[:500]
+            parts.append(f'Parent (context only): "{parent_snippet}"')
+
+        parts.append(f'TARGET: "{comment_text}"')
+
+        return "\n".join(parts)
+
+    # ------------------------------------------------------------------
+    # Content description parsing
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _parse_content_description(text: str) -> str:
+        """
+        Extract meaningful narrative text from a raw content description.
+
+        Strips noise common in YouTube/social descriptions:
+        - "Follow [name]:" blocks and everything after them
+        - Lines starting with ► (hyperlinks)
+        - Lines that are a bare URL
+        - Timestamp chapter markers: "01:08 - Active listening" → "Active listening"
+
+        Returns at most 500 characters of joined clean text.
+        """
+        if not text or not str(text).strip():
+            return ""
+
+        cleaned = []
+        for line in str(text).splitlines():
+            stripped = line.strip()
+
+            # Stop at social-media "Follow" blocks
+            if _RE_FOLLOW_SECTION.match(stripped):
+                break
+
+            # Skip ► link lines
+            if _RE_ARROW_LINK.match(stripped):
+                continue
+
+            # Skip bare URL lines
+            if _RE_URL_ONLY.match(stripped):
+                continue
+
+            # Convert "MM:SS - Chapter label" → keep just the label
+            ts_match = _RE_TIMESTAMP.match(stripped)
+            if ts_match:
+                label = ts_match.group(1).strip()
+                if label:
+                    cleaned.append(label)
+                continue
+
+            if stripped:
+                cleaned.append(stripped)
+
+        return " ".join(cleaned)[:500]
+
+    # ------------------------------------------------------------------
+    # Output validation and reply computation
+    # ------------------------------------------------------------------
+
+    def _validate_result(self, raw: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Validate LLM output against config-defined allowed value sets.
+
+        - Invalid polarity → fail (comment will not be stored)
+        - Invalid intent values → filtered out; if none remain → fail
+        - Invalid confidence → silently corrected to "medium"
+
+        Returns a success dict with cleaned fields, or a failure dict with
+        an explanatory error message.
+        """
+        sentiment_polarity = raw.get("sentiment_polarity")
+
+        if not sentiment_polarity or sentiment_polarity not in self._valid_polarities:
+            return {
+                "success": False,
+                "error": (
+                    f"Invalid sentiment_polarity '{sentiment_polarity}'. "
+                    f"Expected one of: {sorted(self._valid_polarities)}"
+                ),
+            }
+
+        # Normalize intents to a list
+        intents = raw.get("intents", raw.get("intent", []))
+        if isinstance(intents, str):
+            intents = [i.strip() for i in intents.split(",")]
+        if not isinstance(intents, list):
+            intents = []
+
+        valid_intents = [i for i in intents if i in self._valid_intents]
+        if not valid_intents:
+            return {
+                "success": False,
+                "error": (
+                    f"No valid intents in response: {intents}. "
+                    f"Expected values from: {sorted(self._valid_intents)}"
+                ),
+            }
+
+        confidence = raw.get("confidence", "medium")
+        if confidence not in {"high", "medium", "low"}:
+            confidence = "medium"
+
+        return {
+            "success": True,
+            "sentiment_polarity": sentiment_polarity,
+            "intents": valid_intents,
+            "confidence": confidence,
+            "analysis_notes": str(raw.get("analysis_notes", "")).strip(),
+        }
+
+    @staticmethod
+    def _compute_requires_reply(intents: List[str]) -> bool:
+        """
+        Deterministically decide if the comment requires a reply.
+
+        True when the comment contains at least one reply-required intent
+        (question, request, subscription) AND no no-reply intents (humor_sarcasm).
+        This mirrors the reply_policy section of sentiment_analysis_config.json
+        without delegating the decision to the LLM.
+        """
+        intent_set = set(intents)
+        return (
+            bool(intent_set & _REQUIRES_REPLY_INTENTS)
+            and not bool(intent_set & _NO_REPLY_INTENTS)
+        )
+
+    # ------------------------------------------------------------------
+    # Core analysis
+    # ------------------------------------------------------------------
+
+    def analyze_sentiment(
+        self,
+        comment_text: str,
+        content_description: str,
+        parent_comment_text: Optional[str] = None,
+        platform: Optional[str] = None,
+        content_title: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """
+        Call the LLM to classify the TARGET comment's sentiment and intents.
+
+        Args:
+            comment_text: The comment to analyze (translated to English if needed)
+            content_description: Raw content description (will be parsed internally)
+            parent_comment_text: Optional parent comment — context only, max 500 chars
+            platform: Platform name; drives YouTube title-handling logic
+            content_title: YouTube video title (YouTube only)
+
+        Returns:
+            Success dict with sentiment_polarity, intent (comma-separated str),
+            requires_reply, sentiment_confidence, analysis_notes
+            — or a failure dict with an error key.
+        """
+        user_prompt = self._build_user_prompt(
+            comment_text, content_description, parent_comment_text, platform, content_title
+        )
+
+        try:
+            messages = [
+                SystemMessage(content=self._system_prompt),
+                HumanMessage(content=user_prompt),
+            ]
+
+            response = self.llm.invoke(messages)
+            raw = json.loads(response.content)
+
+            validated = self._validate_result(raw)
+            if not validated["success"]:
+                self.log_processing(f"Validation failed: {validated['error']}", "warning")
+                return validated
+
+            requires_reply = self._compute_requires_reply(validated["intents"])
+            intent_str = ", ".join(validated["intents"])
+
+            return {
+                "success": True,
+                "sentiment_polarity": validated["sentiment_polarity"],
+                "intent": intent_str,
+                "requires_reply": requires_reply,
+                "sentiment_confidence": validated["confidence"],
+                "analysis_notes": validated["analysis_notes"],
+            }
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"JSON decode error: {e}", "warning")
+            return {"success": False, "error": f"JSON parse error: {e}"}
+
+        except Exception as e:
+            self.log_processing(f"Sentiment analysis failed: {e}", "error")
+            return {"success": False, "error": str(e)}
+
+    # ------------------------------------------------------------------
+    # Agent interface
+    # ------------------------------------------------------------------
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        return all(field in input_data for field in ("comment_text", "content_description"))
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a comment and return sentiment analysis results merged with
+        the original input fields.
+
+        Args:
+            input_data: Must contain comment_text and content_description.
+                        May contain parent_comment_text, platform, content_title,
+                        and any additional source fields (permalink_url, etc.)
+
+        Returns:
+            Dict with sentiment fields merged on top of original input_data.
+        """
+        try:
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields (comment_text, content_description)",
+                }
+
+            self.log_processing("Analyzing sentiment for comment", "debug")
+
+            analysis_result = self.analyze_sentiment(
+                comment_text=input_data["comment_text"],
+                content_description=input_data["content_description"],
+                parent_comment_text=input_data.get("parent_comment_text"),
+                platform=input_data.get("platform"),
+                content_title=input_data.get("content_title"),
+            )
+
+            result = {
+                "success": analysis_result.get("success", False),
+                "sentiment_polarity": analysis_result.get("sentiment_polarity"),
+                "intent": analysis_result.get("intent"),
+                "requires_reply": analysis_result.get("requires_reply", False),
+                "sentiment_confidence": analysis_result.get("sentiment_confidence"),
+                "analysis_notes": analysis_result.get("analysis_notes", ""),
+            }
+
+            if "error" in analysis_result:
+                result["sentiment_error"] = analysis_result["error"]
+
+            # Preserve all original input fields (e.g. permalink_url, thumbnail_url)
+            for key, value in input_data.items():
+                if key not in result:
+                    result[key] = value
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "sentiment_analysis")

processing_comments/agents/translation_agent.py

@@ -0,0 +1,210 @@
+"""
+Translation Agent
+Translates non-English comments to English using LLM
+"""
+
+from typing import Dict, Any
+import json
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+from agents.base_agent import BaseAgent
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class TranslationAgent(BaseAgent):
+    """
+    Agent that translates text from source language to English.
+    Uses LLM for high-quality, context-aware translation.
+    """
+
+    def __init__(self, config: Dict[str, Any], api_key: str):
+        """
+        Initialize the Translation Agent.
+
+        Args:
+            config: Configuration dictionary
+            api_key: OpenAI API key
+        """
+        super().__init__("TranslationAgent", config)
+        self.api_key = api_key
+        self.llm = ChatOpenAI(
+            model=self.model,
+            temperature=self.temperature,
+            api_key=self.api_key
+        )
+
+    def validate_input(self, input_data: Dict[str, Any]) -> bool:
+        """
+        Validate that input contains required fields.
+
+        Args:
+            input_data: Input dictionary
+
+        Returns:
+            True if valid, False otherwise
+        """
+        required_fields = ["comment_text", "is_english"]
+        return all(field in input_data for field in required_fields)
+
+    def translate_text(self, text: str, source_language: str) -> Dict[str, Any]:
+        """
+        Translate text from source language to English using LLM.
+
+        Args:
+            text: Text to translate
+            source_language: Source language name
+
+        Returns:
+            Dictionary with translation results
+        """
+        system_prompt = """You are a professional translator specializing in social media content related to music and education.
+Translate the given text from the source language to English. The text is a comment on a musical content. 
+Preserve the tone, intent, and any emojis or special characters.
+For informal social media language, maintain the casual tone in translation.
+
+Return your response in JSON format with the following fields:
+- translated_text: The English translation
+- translation_confidence: Your confidence level (high, medium, low)
+- notes: Any important notes about the translation (optional)
+"""
+
+        user_prompt = f"""Translate this {source_language} comment to English:
+
+"{text}"
+
+Return JSON only."""
+
+        try:
+            messages = [
+                SystemMessage(content=system_prompt),
+                HumanMessage(content=user_prompt)
+            ]
+
+            response = self.llm.invoke(messages)
+            result = self._parse_llm_json_response(response.content)
+
+            return {
+                "success": True,
+                "translated_text": result.get("translated_text", text),
+                "translation_confidence": result.get("translation_confidence", "medium"),
+                "translation_notes": result.get("notes", "")
+            }
+
+        except json.JSONDecodeError as e:
+            self.log_processing(f"JSON decode error: {str(e)}", "warning")
+            # Try to extract text from response
+            return {
+                "success": False,
+                "translated_text": text,
+                "translation_confidence": "low",
+                "translation_notes": "JSON parsing failed",
+                "error": str(e)
+            }
+
+        except Exception as e:
+            self.log_processing(f"Translation failed: {str(e)}", "error")
+            return {
+                "success": False,
+                "translated_text": text,
+                "translation_confidence": "low",
+                "translation_notes": "Translation error",
+                "error": str(e)
+            }
+
+    def process(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process comment and translate if needed.
+
+        Args:
+            input_data: Dictionary containing comment data with language info
+
+        Returns:
+            Dictionary with translation results
+        """
+        try:
+            # Validate input
+            if not self.validate_input(input_data):
+                return {
+                    "success": False,
+                    "error": "Invalid input: missing required fields",
+                    "translated_text": input_data.get("comment_text", ""),
+                    "translation_performed": False
+                }
+
+            comment_text = input_data["comment_text"]
+            is_english = input_data["is_english"]
+            source_language = input_data.get("language", "Unknown")
+
+            # If already English, no translation needed
+            if is_english:
+                result = {
+                    "success": True,
+                    "translated_text": comment_text,
+                    "translation_performed": False,
+                    "translation_confidence": "N/A",
+                    "translation_notes": "Original text is English"
+                }
+                self.log_processing("Text is already English, skipping translation", "debug")
+            else:
+                # Perform translation
+                self.log_processing(
+                    f"Translating from {source_language} to English",
+                    "debug"
+                )
+
+                translation_result = self.translate_text(comment_text, source_language)
+
+                result = {
+                    "success": translation_result.get("success", True),
+                    "translated_text": translation_result.get("translated_text", comment_text),
+                    "translation_performed": True,
+                    "translation_confidence": translation_result.get("translation_confidence", "medium"),
+                    "translation_notes": translation_result.get("translation_notes", "")
+                }
+
+                if "error" in translation_result:
+                    result["translation_error"] = translation_result["error"]
+
+            # Preserve all original data
+            for key, value in input_data.items():
+                if key not in result:
+                    result[key] = value
+
+            return result
+
+        except Exception as e:
+            return self.handle_error(e, "translation")
+
+    def _parse_llm_json_response(self, response_content: str) -> Dict[str, Any]:
+        """
+        Parse LLM response that may contain JSON wrapped in markdown code blocks.
+
+        Args:
+            response_content: Raw response content from LLM
+
+        Returns:
+            Parsed JSON dictionary
+
+        Raises:
+            json.JSONDecodeError: If JSON cannot be parsed
+        """
+        content = response_content.strip()
+
+        # Check if response is wrapped in markdown code block
+        if content.startswith("```json"):
+            # Remove ```json prefix and ``` suffix
+            content = content[7:]  # Remove ```json
+            if content.endswith("```"):
+                content = content[:-3]  # Remove trailing ```
+            content = content.strip()
+        elif content.startswith("```"):
+            # Remove generic ``` code block
+            content = content[3:]
+            if content.endswith("```"):
+                content = content[:-3]
+            content = content.strip()
+
+        # Parse the cleaned JSON
+        return json.loads(content)

processing_comments/config_files/data_sources_config.json

@@ -0,0 +1,56 @@
+{
+    "data_sources": {
+        "social_media": {
+            "name": "Social Media Comments",
+            "description": "Comments from external social media platforms (Facebook, Instagram, YouTube, etc.)",
+            "enabled": true,
+            "sql_query_file": "sql/fetch_comments.sql",
+            "output_config": {
+                "table_name": "COMMENT_SENTIMENT_FEATURES",
+                "database": "SOCIAL_MEDIA_DB",
+                "schema": "ML_FEATURES"
+            },
+            "source_columns": {
+                "comment_sk": "COMMENT_SK",
+                "comment_id": "COMMENT_ID",
+                "comment_text": "COMMENT_TEXT",
+                "parent_comment_id": "PARENT_COMMENT_ID",
+                "parent_comment_text": "PARENT_COMMENT_TEXT",
+                "platform": "PLATFORM",
+                "content_description": "CONTENT_DESCRIPTION"
+            }
+        },
+        "musora_comments": {
+            "name": "Musora Internal Comments",
+            "description": "Comments from Musora internal applications",
+            "enabled": true,
+            "sql_query_file": "sql/fetch_musora_comments.sql",
+            "output_config": {
+                "table_name": "MUSORA_COMMENT_SENTIMENT_FEATURES",
+                "database": "SOCIAL_MEDIA_DB",
+                "schema": "ML_FEATURES"
+            },
+            "source_columns": {
+                "comment_sk": "COMMENT_SK (generated via HASH)",
+                "comment_id": "COMMENT_ID",
+                "comment_text": "COMMENT_TEXT (aliased from MESSAGE)",
+                "parent_comment_id": "PARENT_COMMENT_ID",
+                "parent_comment_text": "PARENT_COMMENT_TEXT",
+                "platform": "PLATFORM",
+                "content_description": "CONTENT_DESCRIPTION (aliased from CONTENT_PROFILE)",
+                "author_id": "AUTHOR_ID (aliased from USER_ID)",
+                "permalink_url": "PERMALINK_URL (aliased from WEB_URL_PATH)",
+                "thumbnail_url": "THUMBNAIL_URL"
+            },
+            "additional_fields": [
+                "PERMALINK_URL",
+                "THUMBNAIL_URL"
+            ]
+        }
+    },
+    "processing": {
+        "default_limit": 10000,
+        "enable_parent_context": true,
+        "parent_context_description": "When a comment is a reply, include the parent comment text for better sentiment analysis context"
+    }
+}

processing_comments/config_files/sentiment_analysis_config.json

@@ -0,0 +1,96 @@
+{
+    "sentiment_polarity": {
+        "categories": [
+            {
+                "value": "very_positive",
+                "label": "Very Positive",
+                "description": "Extremely enthusiastic, excited, deeply grateful, or highly satisfied"
+            },
+            {
+                "value": "positive",
+                "label": "Positive",
+                "description": "Generally positive, appreciative, supportive, or encouraging"
+            },
+            {
+                "value": "neutral",
+                "label": "Neutral",
+                "description": "Factual, informational, balanced, or lacking clear emotional tone"
+            },
+            {
+                "value": "negative",
+                "label": "Negative",
+                "description": "Disappointed, critical, frustrated, or mildly dissatisfied"
+            },
+            {
+                "value": "very_negative",
+                "label": "Very Negative",
+                "description": "Highly critical, angry, abusive, or extremely dissatisfied"
+            }
+        ]
+    },
+    "intent": {
+        "categories": [
+            {
+                "value": "praise",
+                "label": "Praise",
+                "description": "Compliments, thanks, admiration, excitement, and similar positive expressions"
+            },
+            {
+                "value": "question",
+                "label": "Question",
+                "description": "Information seeking (e.g., 'what scale?', 'when's it out?', How to get account?)"
+            },
+            {
+                "value": "request",
+                "label": "Request",
+                "description": "Asking for something actionable (tutorial, feature, sheet music, etc.)"
+            },
+            {
+                "value": "feedback_negative",
+                "label": "Negative Feedback",
+                "description": "Critical feedback about the content or issues (mixing, performance, composition) without abuse"
+            },
+            {
+                "value": "suggestion",
+                "label": "Suggestion",
+                "description": "Constructive ideas/improvements (e.g., 'try slower tempo', 'add captions')"
+            },
+            {
+                "value": "humor_sarcasm",
+                "label": "Humor/Sarcasm",
+                "description": "Joking, teasing, memes, irony (non-toxic)"
+            },
+            {
+                "value": "off_topic",
+                "label": "Off Topic",
+                "description": "Unrelated chatter or unclear/no discernible intent"
+            },
+            {
+                "value": "spam_selfpromo",
+                "label": "Spam/Self-Promotion",
+                "description": "Ads, links, promos, scams"
+            },
+            {
+                "value": "subscription",
+                "label": "Subscription",
+                "description": "Questions about subscribing (e.g., 'How do I subscribe?', 'What's the cost?') or requests to unsubscribe/cancel (e.g., 'I want to cancel', 'How to unsubscribe?')"
+            }
+        ]
+    },
+    "reply_policy": {
+        "requires_reply_intents": ["question", "request", "subscription"],
+        "not_include": ["humor_sarcasm"],
+        "description": "Comments with these intents should be flagged for reply"
+    },
+    "intent_settings": {
+        "multi_label": true,
+        "description": "Intent can have multiple labels as a comment can express multiple intents",
+        "rhetorical_sarcasm_handling": true,
+        "rhetorical_sarcasm_description": "System differentiates between genuine questions/suggestions/requests and rhetorical/sarcastic ones"
+    },
+    "analysis_notes_policy": {
+        "max_length": "1-2 sentences",
+        "include_topics": true,
+        "description": "Concise notes including key topics/highlights not covered by other categories for future summarization"
+    }
+}

processing_comments/config_files/sentiment_config.json

@@ -0,0 +1,49 @@
+{
+    "LLM_models": ["gpt-5-nano", "gpt-4o-mini"],
+    "reasoning": ["gpt-5-nano"],
+
+    "agents": {
+        "language_detection": {
+            "name": "LanguageDetectionAgent",
+            "model": "gpt-5-nano",
+            "temperature": 0.0,
+            "max_retries": 3,
+            "description": "Detects language of comments and identifies non-English content"
+        },
+        "translation": {
+            "name": "TranslationAgent",
+            "model": "gpt-5-nano",
+            "temperature": 0.3,
+            "max_retries": 3,
+            "description": "Translates non-English comments to English"
+        },
+        "sentiment_analysis": {
+            "name": "SentimentAnalysisAgent",
+            "model": "gpt-5-nano",
+            "temperature": 0.0,
+            "max_retries": 3,
+            "description": "Analyzes sentiment polarity, intent, and determines if reply is needed"
+        }
+    },
+
+    "workflow": {
+        "description": "Batch size is calculated dynamically based on number of workers (min: 20, max: 1000)",
+        "parallel_processing": {
+            "enabled": true,
+            "worker_calculation": "CPU count - 2, max 5 workers",
+            "min_batch_size": 20,
+            "max_batch_size": 1000
+        }
+    },
+
+    "snowflake": {
+        "output_table": "COMMENT_SENTIMENT_FEATURES",
+        "database": "SOCIAL_MEDIA_DB",
+        "schema": "ML_FEATURES"
+    },
+
+    "default_language": "English"
+}
+
+
+

processing_comments/main.py

@@ -0,0 +1,572 @@
+"""
+Main execution script for comment processing workflow.
+Orchestrates data fetching, processing, and storage using agentic workflow.
+Supports parallel processing with multiprocessing for improved performance.
+Supports multiple data sources (social media and Musora internal comments).
+"""
+
+import json
+import os
+import logging
+import argparse
+from datetime import datetime
+import pandas as pd
+from dotenv import load_dotenv
+from multiprocessing import Pool, cpu_count, Manager
+from functools import partial
+import traceback
+from typing import Dict, Any, List
+
+from SnowFlakeConnection import SnowFlakeConn
+from workflow.comment_processor import CommentProcessingWorkflow
+
+# Get the directory where this script is located
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Load environment variables from root directory (parent of processing_comments)
+ROOT_DIR = os.path.dirname(SCRIPT_DIR)
+load_dotenv(os.path.join(ROOT_DIR, '.env'))
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler(os.path.join(SCRIPT_DIR, 'logs', f'comment_processing_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log')),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+
+def calculate_optimal_batch_size(total_comments: int, num_workers: int, min_batch: int = 20, max_batch: int = 100) -> int:
+    """
+    Calculate optimal batch size based on total comments and number of workers.
+
+    Args:
+        total_comments: Total number of comments to process
+        num_workers: Number of parallel workers
+        min_batch: Minimum batch size (default: 20)
+        max_batch: Maximum batch size (default: 1000)
+
+    Returns:
+        Optimal batch size
+    """
+    if total_comments <= min_batch:
+        return total_comments
+
+    # Calculate batch size to distribute work evenly among workers
+    batch_size = total_comments // num_workers
+
+    # Apply constraints
+    batch_size = max(min_batch, min(max_batch, batch_size))
+
+    return batch_size
+
+
+def process_batch_worker(batch_data: tuple) -> dict:
+    """
+    Worker function to process a single batch of comments.
+    This function runs in a separate process.
+
+    Args:
+        batch_data: Tuple containing (batch_num, batch_comments, config, api_key, overwrite_first_batch, data_source_config)
+
+    Returns:
+        Dictionary with batch statistics and results
+    """
+    batch_num, batch_comments, config, api_key, overwrite_first_batch, data_source_config = batch_data
+
+    # Configure logging for this worker
+    worker_logger = logging.getLogger(f"Worker-{batch_num}")
+
+    try:
+        worker_logger.info(f"Batch {batch_num}: Starting processing of {len(batch_comments)} comments")
+
+        # Initialize Snowflake connection for this worker
+        snowflake = SnowFlakeConn()
+
+        # Initialize workflow for this worker
+        workflow = CommentProcessingWorkflow(config, api_key)
+
+        # Process comments through workflow
+        results = workflow.process_batch(batch_comments)
+
+        # Convert to DataFrame
+        results_df = pd.DataFrame(results)
+
+        # Filter successful results
+        initial_count = len(results_df)
+        df_successful = results_df[results_df['success'] == True].copy()
+        filtered_count = initial_count - len(df_successful)
+
+        worker_logger.info(f"Batch {batch_num}: Processed {initial_count} comments, {len(df_successful)} successful")
+
+        # Prepare output data with base columns
+        output_columns = {
+            'comment_sk': 'COMMENT_SK',
+            'comment_id': 'COMMENT_ID',
+            'comment_text': 'ORIGINAL_TEXT',
+            'platform': 'PLATFORM',
+            'comment_timestamp': 'COMMENT_TIMESTAMP',
+            'author_name': 'AUTHOR_NAME',
+            'author_id': 'AUTHOR_ID',
+            'parent_comment_id': 'PARENT_COMMENT_ID',
+            'parent_comment_text': 'PARENT_COMMENT_TEXT',
+            'content_sk': 'CONTENT_SK',
+            'content_id': 'CONTENT_ID',
+            'content_description': 'CONTENT_DESCRIPTION',
+            'channel_sk': 'CHANNEL_SK',
+            'channel_name': 'CHANNEL_NAME',
+            'channel_display_name': 'CHANNEL_DISPLAY_NAME',
+            'language': 'DETECTED_LANGUAGE',
+            'language_code': 'LANGUAGE_CODE',
+            'is_english': 'IS_ENGLISH',
+            'language_confidence': 'LANGUAGE_CONFIDENCE',
+            'detection_method': 'DETECTION_METHOD',
+            'has_text': 'HAS_TEXT',
+            'translated_text': 'TRANSLATED_TEXT',
+            'translation_performed': 'TRANSLATION_PERFORMED',
+            'translation_confidence': 'TRANSLATION_CONFIDENCE',
+            'translation_notes': 'TRANSLATION_NOTES',
+            'sentiment_polarity': 'SENTIMENT_POLARITY',
+            'intent': 'INTENT',
+            'requires_reply': 'REQUIRES_REPLY',
+            'sentiment_confidence': 'SENTIMENT_CONFIDENCE',
+            'analysis_notes': 'ANALYSIS_NOTES',
+            'success': 'PROCESSING_SUCCESS'
+        }
+
+        # Add data source-specific columns if present
+        if 'additional_fields' in data_source_config:
+            for field in data_source_config['additional_fields']:
+                field_lower = field.lower()
+                output_columns[field_lower] = field
+            worker_logger.debug(f"Batch {batch_num}: Added {len(data_source_config['additional_fields'])} additional fields")
+
+        output_df = pd.DataFrame()
+        for source_col, target_col in output_columns.items():
+            if source_col in df_successful.columns:
+                output_df[target_col] = df_successful[source_col]
+            else:
+                output_df[target_col] = None
+                # Log missing columns for debugging
+                if source_col in ['permalink_url', 'thumbnail_url']:
+                    worker_logger.warning(f"Batch {batch_num}: Column '{source_col}' not found in DataFrame. Available columns: {list(df_successful.columns)}")
+
+        # Add processing metadata
+        output_df['PROCESSED_AT'] = datetime.now()
+        output_df['WORKFLOW_VERSION'] = '1.0'
+
+        # Store results to Snowflake
+        if len(output_df) > 0:
+            # Use data source-specific output configuration
+            table_name = data_source_config['output_config']['table_name']
+            database = data_source_config['output_config']['database']
+            schema = data_source_config['output_config']['schema']
+
+            # Only the first batch should overwrite if requested
+            overwrite = overwrite_first_batch and batch_num == 1
+
+            snowflake.store_df_to_snowflake(
+                table_name=table_name,
+                dataframe=output_df,
+                database=database,
+                schema=schema,
+                overwrite=overwrite
+            )
+
+            worker_logger.info(f"Batch {batch_num}: Stored {len(output_df)} records to Snowflake ({table_name})")
+        else:
+            worker_logger.warning(f"Batch {batch_num}: No successful records to store")
+
+        # Close Snowflake connection
+        snowflake.close_connection()
+
+        # Calculate statistics
+        translations = output_df['TRANSLATION_PERFORMED'].sum() if 'TRANSLATION_PERFORMED' in output_df.columns else 0
+        non_english = (~output_df['IS_ENGLISH']).sum() if 'IS_ENGLISH' in output_df.columns else 0
+        requires_reply = output_df['REQUIRES_REPLY'].sum() if 'REQUIRES_REPLY' in output_df.columns else 0
+
+        return {
+            'batch_num': batch_num,
+            'success': True,
+            'total_processed': initial_count,
+            'total_stored': len(output_df),
+            'failed_count': filtered_count,
+            'translations': int(translations),
+            'non_english': int(non_english),
+            'requires_reply': int(requires_reply),
+            'error': None
+        }
+
+    except Exception as e:
+        error_msg = f"Batch {batch_num} failed: {str(e)}"
+        worker_logger.error(error_msg)
+        worker_logger.error(traceback.format_exc())
+
+        return {
+            'batch_num': batch_num,
+            'success': False,
+            'total_processed': len(batch_comments),
+            'total_stored': 0,
+            'failed_count': len(batch_comments),
+            'translations': 0,
+            'non_english': 0,
+            'requires_reply': 0,
+            'error': error_msg
+        }
+
+
+class CommentProcessor:
+    """
+    Main processor class that orchestrates the entire workflow.
+    Supports multiple data sources (social media and Musora internal comments).
+    """
+
+    def __init__(self, config_path: str = None, data_sources_config_path: str = None):
+        """
+        Initialize the comment processor.
+
+        Args:
+            config_path: Path to configuration file (default: config_files/sentiment_config.json relative to script)
+            data_sources_config_path: Path to data sources config (default: config_files/data_sources_config.json)
+        """
+        # Set default config path if not provided
+        if config_path is None:
+            config_path = os.path.join(SCRIPT_DIR, 'config_files', 'sentiment_config.json')
+
+        if data_sources_config_path is None:
+            data_sources_config_path = os.path.join(SCRIPT_DIR, 'config_files', 'data_sources_config.json')
+
+        # Load configuration
+        with open(config_path, 'r') as f:
+            self.config = json.load(f)
+
+        # Load data sources configuration
+        with open(data_sources_config_path, 'r') as f:
+            self.data_sources_config = json.load(f)
+
+        # Initialize Snowflake connection
+        self.snowflake = SnowFlakeConn()
+
+        # Get OpenAI API key
+        self.api_key = os.getenv("OPENAI_API_KEY")
+        if not self.api_key:
+            raise ValueError("OPENAI_API_KEY not found in environment variables")
+
+        # Initialize workflow
+        self.workflow = CommentProcessingWorkflow(self.config, self.api_key)
+
+        logger.info("CommentProcessor initialized successfully")
+
+    def get_enabled_data_sources(self) -> List[Dict[str, Any]]:
+        """
+        Get list of enabled data sources from configuration.
+
+        Returns:
+            List of enabled data source configurations
+        """
+        enabled_sources = []
+        for source_key, source_config in self.data_sources_config['data_sources'].items():
+            if source_config.get('enabled', True):
+                enabled_sources.append({
+                    'key': source_key,
+                    'config': source_config
+                })
+        return enabled_sources
+
+    def fetch_comments(self, data_source_key: str, limit: int = None) -> pd.DataFrame:
+        """
+        Fetch comments from Snowflake using the SQL query for a specific data source.
+
+        Args:
+            data_source_key: Key identifying the data source (e.g., 'social_media', 'musora_comments')
+            limit: Optional limit on number of comments to fetch
+
+        Returns:
+            DataFrame containing comment data
+        """
+        data_source_config = self.data_sources_config['data_sources'][data_source_key]
+        source_name = data_source_config['name']
+
+        logger.info(f"Fetching comments from {source_name}...")
+
+        # Read SQL query
+        sql_file = data_source_config['sql_query_file']
+        sql_path = os.path.join(SCRIPT_DIR, sql_file)
+        with open(sql_path, 'r') as f:
+            query = f.read()
+
+        # Add limit if specified
+        if limit:
+            query = query.rstrip(';') + f"\nLIMIT {limit};"
+
+        # Execute query
+        df = self.snowflake.run_read_query(query, f"{source_name} comments")
+
+        logger.info(f"Fetched {len(df)} comments from {source_name}")
+
+        # Normalize column names to lowercase for consistent processing
+        df.columns = df.columns.str.lower()
+
+        # Additional validation: filter out any empty comments that might have slipped through
+        if 'comment_text' in df.columns:
+            initial_count = len(df)
+            df = df[df['comment_text'].notna() & (df['comment_text'].str.strip() != '')]
+            filtered_count = initial_count - len(df)
+            if filtered_count > 0:
+                logger.info(f"Filtered out {filtered_count} empty comments in post-processing")
+
+        logger.info(f"Final count: {len(df)} non-empty comments")
+        return df
+
+    def calculate_num_workers(self) -> int:
+        """
+        Calculate the number of parallel workers to use.
+        Uses CPU count - 2, with a maximum of 5 workers.
+
+        Returns:
+            Number of workers
+        """
+        num_cpus = cpu_count()
+        num_workers = max(1, min(5, num_cpus - 2))
+        logger.info(f"Using {num_workers} parallel workers (CPU count: {num_cpus})")
+        return num_workers
+
+    def process_comments_parallel(self, df: pd.DataFrame, data_source_config: Dict[str, Any], overwrite: bool = False) -> dict:
+        """
+        Process comments through the agentic workflow using parallel processing.
+
+        Args:
+            df: DataFrame containing raw comment data
+            data_source_config: Configuration for the data source being processed
+            overwrite: Whether to overwrite existing Snowflake table
+
+        Returns:
+            Dictionary with aggregated statistics
+        """
+        # Convert DataFrame to list of dictionaries
+        comments = df.to_dict('records')
+        total_comments = len(comments)
+
+        logger.info(f"Processing {total_comments} comments using parallel processing...")
+
+        # Calculate number of workers
+        num_workers = self.calculate_num_workers()
+
+        # Calculate optimal batch size
+        batch_size = calculate_optimal_batch_size(total_comments, num_workers)
+        logger.info(f"Batch size: {batch_size} (min: 20, max: 100)")
+
+        # Create batches
+        batches = []
+        for i in range(0, total_comments, batch_size):
+            batch = comments[i:i + batch_size]
+            batch_num = (i // batch_size) + 1
+            batches.append((batch_num, batch, self.config, self.api_key, overwrite, data_source_config))
+
+        total_batches = len(batches)
+        logger.info(f"Split into {total_batches} batches")
+
+        # Process batches in parallel
+        with Pool(processes=num_workers) as pool:
+            results = pool.map(process_batch_worker, batches)
+
+        # Aggregate statistics
+        total_processed = sum(r['total_processed'] for r in results)
+        total_stored = sum(r['total_stored'] for r in results)
+        failed_count = sum(r['failed_count'] for r in results)
+        translations = sum(r['translations'] for r in results)
+        non_english = sum(r['non_english'] for r in results)
+        requires_reply = sum(r['requires_reply'] for r in results)
+
+        # Count failed batches
+        failed_batches = [r for r in results if not r['success']]
+        if failed_batches:
+            logger.error(f"{len(failed_batches)} batch(es) failed:")
+            for fb in failed_batches:
+                logger.error(f"  Batch {fb['batch_num']}: {fb['error']}")
+
+        return {
+            'total_processed': total_processed,
+            'total_stored': total_stored,
+            'failed_count': failed_count,
+            'translations': translations,
+            'non_english': non_english,
+            'requires_reply': requires_reply,
+            'failed_batches': len(failed_batches)
+        }
+
+    def process_comments_sequential(self, df: pd.DataFrame, data_source_config: Dict[str, Any], overwrite: bool = False) -> dict:
+        """
+        Process comments through the agentic workflow sequentially (for debugging).
+
+        Args:
+            df: DataFrame containing raw comment data
+            data_source_config: Configuration for the data source being processed
+            overwrite: Whether to overwrite existing Snowflake table
+
+        Returns:
+            Dictionary with aggregated statistics
+        """
+        logger.info(f"Processing {len(df)} comments using sequential processing (debug mode)...")
+
+        # Convert DataFrame to list of dictionaries
+        comments = df.to_dict('records')
+
+        # Process as a single batch
+        batch_data = (1, comments, self.config, self.api_key, overwrite, data_source_config)
+        result = process_batch_worker(batch_data)
+
+        return {
+            'total_processed': result['total_processed'],
+            'total_stored': result['total_stored'],
+            'failed_count': result['failed_count'],
+            'translations': result['translations'],
+            'non_english': result['non_english'],
+            'requires_reply': result['requires_reply'],
+            'failed_batches': 0 if result['success'] else 1
+        }
+
+    def run(self, limit: int = None, overwrite: bool = False, sequential: bool = False, data_source_filter: str = None):
+        """
+        Run the complete processing pipeline for all enabled data sources.
+
+        Args:
+            limit: Optional limit on number of comments to process per data source
+            overwrite: Whether to overwrite existing Snowflake table
+            sequential: If True, use sequential processing instead of parallel (for debugging)
+            data_source_filter: Optional filter to process only a specific data source
+        """
+        try:
+            logger.info("=" * 80)
+            logger.info("Starting Comment Processing Workflow")
+            if sequential:
+                logger.info("Mode: SEQUENTIAL (Debug Mode)")
+            else:
+                logger.info("Mode: PARALLEL")
+            logger.info("=" * 80)
+
+            # Get enabled data sources
+            enabled_sources = self.get_enabled_data_sources()
+
+            if data_source_filter:
+                enabled_sources = [s for s in enabled_sources if s['key'] == data_source_filter]
+                if not enabled_sources:
+                    logger.error(f"Data source '{data_source_filter}' not found or not enabled")
+                    return
+
+            logger.info(f"Processing {len(enabled_sources)} data source(s)")
+
+            # Process each data source
+            for source_info in enabled_sources:
+                source_key = source_info['key']
+                source_config = source_info['config']
+                source_name = source_config['name']
+
+                logger.info("=" * 80)
+                logger.info(f"Processing Data Source: {source_name}")
+                logger.info("=" * 80)
+
+                # Step 1: Fetch comments
+                df_comments = self.fetch_comments(data_source_key=source_key, limit=limit)
+
+                if df_comments.empty:
+                    logger.warning(f"No comments to process from {source_name}")
+                    continue
+
+                # Step 2: Process comments through workflow (parallel or sequential)
+                start_time = datetime.now()
+
+                if sequential:
+                    stats = self.process_comments_sequential(df_comments, source_config, overwrite=overwrite)
+                else:
+                    stats = self.process_comments_parallel(df_comments, source_config, overwrite=overwrite)
+
+                end_time = datetime.now()
+                processing_time = (end_time - start_time).total_seconds()
+
+                # Summary statistics
+                logger.info("=" * 80)
+                logger.info(f"Processing Summary for {source_name}:")
+                logger.info(f"  Processing Mode: {'Sequential' if sequential else 'Parallel'}")
+                logger.info(f"  Output Table: {source_config['output_config']['table_name']}")
+                logger.info(f"  Total comments processed: {stats['total_processed']}")
+                logger.info(f"  Successfully stored: {stats['total_stored']}")
+                logger.info(f"  Failed sentiment analysis (not stored): {stats['failed_count']}")
+                if stats.get('failed_batches', 0) > 0:
+                    logger.info(f"  Failed batches: {stats['failed_batches']}")
+                logger.info(f"  Non-English comments: {stats['non_english']}")
+                logger.info(f"  Translations performed: {stats['translations']}")
+                logger.info(f"  Comments requiring reply: {stats['requires_reply']}")
+                logger.info(f"  Processing time: {processing_time:.2f} seconds")
+                logger.info(f"  Average time per comment: {processing_time / stats['total_processed']:.2f} seconds")
+                logger.info("=" * 80)
+
+        except Exception as e:
+            logger.error(f"Error in workflow execution: {str(e)}", exc_info=True)
+            raise
+
+        finally:
+            # Close main Snowflake connection (workers have their own connections)
+            self.snowflake.close_connection()
+            logger.info("Snowflake connection closed")
+
+
+def main():
+    """
+    Main entry point for the script.
+    """
+    parser = argparse.ArgumentParser(
+        description="Process comments with language detection, translation, and sentiment analysis from multiple data sources"
+    )
+    parser.add_argument(
+        '--limit',
+        type=int,
+        default=5000,
+        help='Limit number of comments to process per data source (default: 10000)'
+    )
+    parser.add_argument(
+        '--overwrite',
+        action='store_true',
+        default=False,
+        help='Overwrite existing Snowflake table (default: False, appends new records)'
+    )
+    parser.add_argument(
+        '--config',
+        type=str,
+        default=None,
+        help='Path to configuration file (default: config_files/sentiment_config.json relative to script)'
+    )
+    parser.add_argument(
+        '--sequential',
+        action='store_true',
+        default=False,
+        help='Use sequential processing instead of parallel (for debugging)'
+    )
+    parser.add_argument(
+        '--data-source',
+        type=str,
+        default=None,
+        help='Process only a specific data source (e.g., social_media, musora_comments). If not specified, all enabled sources are processed.'
+    )
+
+    args = parser.parse_args()
+
+    # Create logs directory if it doesn't exist
+    logs_dir = os.path.join(SCRIPT_DIR, 'logs')
+    os.makedirs(logs_dir, exist_ok=True)
+
+    # Initialize and run processor
+    processor = CommentProcessor(config_path=args.config)
+    processor.run(
+        limit=args.limit,
+        overwrite=args.overwrite,
+        sequential=args.sequential,
+        data_source_filter=args.data_source
+    )
+
+
+if __name__ == "__main__":
+    main()

processing_comments/requirements.txt

@@ -0,0 +1,10 @@
+snowflake-snowpark-python>=1.0.0
+pandas>=1.3.0
+python-dotenv>=0.19.0
+openai>=1.0.0
+argparse
+langchain>=0.1.0
+langchain-openai>=0.0.5
+langgraph>=0.0.20
+lingua-language-detector>=2.0.0
+pydantic>=2.0.0

processing_comments/sql/create_ml_features_table.sql

@@ -0,0 +1,127 @@
+-- Create table in ML_FEATURES schema to store comment sentiment analysis results
+-- This table stores the output from the language detection, translation, and sentiment analysis workflow
+
+USE DATABASE SOCIAL_MEDIA_DB;
+USE SCHEMA ML_FEATURES;
+
+CREATE TABLE IF NOT EXISTS COMMENT_SENTIMENT_FEATURES (
+    -- Primary identifiers
+    COMMENT_SK NUMBER(38,0) NOT NULL COMMENT 'Surrogate key from FACT_COMMENTS',
+    COMMENT_ID VARCHAR(16777216) NOT NULL COMMENT 'Platform comment ID',
+    ORIGINAL_TEXT VARCHAR(16777216) COMMENT 'Original comment text',
+    PLATFORM VARCHAR(16777216) COMMENT 'Social platform',
+    COMMENT_TIMESTAMP TIMESTAMP_NTZ(9) COMMENT 'When comment was posted',
+    AUTHOR_NAME VARCHAR(16777216) COMMENT 'Commenter name',
+    AUTHOR_ID VARCHAR(16777216) COMMENT 'Platform user ID',
+
+    -- Parent comment information
+    PARENT_COMMENT_ID VARCHAR(16777216) COMMENT 'ID of parent comment if this is a reply',
+    PARENT_COMMENT_TEXT VARCHAR(16777216) COMMENT 'Text of parent comment for context',
+
+    -- Content references
+    CONTENT_SK NUMBER(38,0) COMMENT 'Foreign key to content',
+    CONTENT_ID VARCHAR(16777216) COMMENT 'Platform content ID',
+    CONTENT_DESCRIPTION VARCHAR(16777216) COMMENT 'Content description/message',
+
+    -- Channel references
+    CHANNEL_SK NUMBER(38,0) COMMENT 'Foreign key to channel',
+    CHANNEL_NAME VARCHAR(16777216) COMMENT 'Brand/channel name',
+    CHANNEL_DISPLAY_NAME VARCHAR(16777216) COMMENT 'Channel display name',
+
+    -- Language detection features
+    DETECTED_LANGUAGE VARCHAR(100) COMMENT 'Detected language name (e.g., English, Spanish)',
+    LANGUAGE_CODE VARCHAR(10) COMMENT 'ISO 639-1 language code (e.g., en, es)',
+    IS_ENGLISH BOOLEAN COMMENT 'True if comment is in English',
+    LANGUAGE_CONFIDENCE VARCHAR(20) COMMENT 'Confidence level: high, medium, low',
+    DETECTION_METHOD VARCHAR(50) COMMENT 'Method used: library, llm, or default',
+    HAS_TEXT BOOLEAN COMMENT 'True if comment has textual content (not just emojis)',
+
+    -- Translation features
+    TRANSLATED_TEXT VARCHAR(16777216) COMMENT 'English translation (or original if already English)',
+    TRANSLATION_PERFORMED BOOLEAN COMMENT 'True if translation was performed',
+    TRANSLATION_CONFIDENCE VARCHAR(20) COMMENT 'Translation confidence level',
+    TRANSLATION_NOTES VARCHAR(16777216) COMMENT 'Notes about translation',
+
+    -- Sentiment analysis features
+    SENTIMENT_POLARITY VARCHAR(20) COMMENT 'Sentiment: very_positive, positive, neutral, negative, very_negative',
+    INTENT VARCHAR(500) COMMENT 'Multi-label intents (comma-separated): praise, question, request, feedback_negative, suggestion, humor_sarcasm, off_topic, spam_selfpromo',
+    REQUIRES_REPLY BOOLEAN COMMENT 'True if comment requires a response (genuine questions/requests only)',
+    SENTIMENT_CONFIDENCE VARCHAR(20) COMMENT 'Sentiment analysis confidence: high, medium, low',
+    ANALYSIS_NOTES VARCHAR(16777216) COMMENT 'Concise notes with key topics/highlights for summarization',
+
+    -- Processing metadata
+    PROCESSING_SUCCESS BOOLEAN COMMENT 'True if processing completed successfully',
+    PROCESSING_ERRORS VARCHAR(16777216) COMMENT 'Any errors encountered during processing',
+    PROCESSED_AT TIMESTAMP_NTZ(9) COMMENT 'When this record was processed',
+    WORKFLOW_VERSION VARCHAR(20) COMMENT 'Version of the processing workflow',
+
+    -- Audit fields
+    CREATED_AT TIMESTAMP_NTZ(9) DEFAULT CURRENT_TIMESTAMP() COMMENT 'Record creation time',
+    UPDATED_AT TIMESTAMP_NTZ(9) DEFAULT CURRENT_TIMESTAMP() COMMENT 'Record update time'
+)
+COMMENT='ML Features table for language detection, translation, and sentiment analysis results from social media comments';
+
+-- Create indexes for common queries
+-- Note: Snowflake automatically optimizes queries, but we can define clustering keys
+ALTER TABLE COMMENT_SENTIMENT_FEATURES CLUSTER BY (COMMENT_TIMESTAMP, CHANNEL_NAME);
+
+-- Create view for comments requiring reply
+CREATE OR REPLACE VIEW VW_COMMENTS_REQUIRING_REPLY AS
+SELECT
+    COMMENT_SK,
+    COMMENT_ID,
+    ORIGINAL_TEXT,
+    TRANSLATED_TEXT,
+    PARENT_COMMENT_ID,
+    PARENT_COMMENT_TEXT,
+    INTENT,
+    SENTIMENT_POLARITY,
+    SENTIMENT_CONFIDENCE,
+    CHANNEL_NAME,
+    AUTHOR_NAME,
+    COMMENT_TIMESTAMP,
+    PLATFORM,
+    CONTENT_DESCRIPTION
+FROM COMMENT_SENTIMENT_FEATURES
+WHERE REQUIRES_REPLY = TRUE
+    AND PROCESSING_SUCCESS = TRUE
+ORDER BY COMMENT_TIMESTAMP DESC;
+
+-- Create view for sentiment distribution
+CREATE OR REPLACE VIEW VW_SENTIMENT_DISTRIBUTION AS
+SELECT
+    CHANNEL_NAME,
+    SENTIMENT_POLARITY,
+    INTENT,
+    COUNT(*) AS COMMENT_COUNT,
+    COUNT(CASE WHEN REQUIRES_REPLY THEN 1 END) AS REPLIES_NEEDED,
+    COUNT(CASE WHEN PARENT_COMMENT_ID IS NOT NULL THEN 1 END) AS REPLY_COMMENTS,
+    AVG(CASE WHEN SENTIMENT_CONFIDENCE = 'high' THEN 3
+             WHEN SENTIMENT_CONFIDENCE = 'medium' THEN 2
+             WHEN SENTIMENT_CONFIDENCE = 'low' THEN 1
+             ELSE 0 END) AS AVG_CONFIDENCE_SCORE,
+    MAX(PROCESSED_AT) AS LAST_PROCESSED
+FROM COMMENT_SENTIMENT_FEATURES
+WHERE PROCESSING_SUCCESS = TRUE
+GROUP BY CHANNEL_NAME, SENTIMENT_POLARITY, INTENT
+ORDER BY CHANNEL_NAME, COMMENT_COUNT DESC;
+
+-- Create view for non-English comments
+CREATE OR REPLACE VIEW VW_NON_ENGLISH_COMMENTS AS
+SELECT
+    COMMENT_SK,
+    COMMENT_ID,
+    ORIGINAL_TEXT,
+    DETECTED_LANGUAGE,
+    LANGUAGE_CODE,
+    TRANSLATED_TEXT,
+    TRANSLATION_CONFIDENCE,
+    SENTIMENT_POLARITY,
+    INTENT,
+    CHANNEL_NAME,
+    COMMENT_TIMESTAMP,
+    PLATFORM
+FROM COMMENT_SENTIMENT_FEATURES
+WHERE IS_ENGLISH = FALSE
+    AND PROCESSING_SUCCESS = TRUE
+ORDER BY COMMENT_TIMESTAMP DESC;

processing_comments/sql/create_musora_ml_features_table.sql

@@ -0,0 +1,135 @@
+-- Create table in ML_FEATURES schema to store Musora comment sentiment analysis results
+-- This table stores the output from the language detection, translation, and sentiment analysis workflow
+-- Schema matches COMMENT_SENTIMENT_FEATURES with additional Musora-specific fields
+
+USE DATABASE SOCIAL_MEDIA_DB;
+USE SCHEMA ML_FEATURES;
+
+CREATE TABLE IF NOT EXISTS MUSORA_COMMENT_SENTIMENT_FEATURES (
+    -- Primary identifiers
+    COMMENT_SK NUMBER(38,0) NOT NULL COMMENT 'Generated surrogate key (hash of COMMENT_ID and PLATFORM)',
+    COMMENT_ID VARCHAR(16777216) NOT NULL COMMENT 'Musora comment ID',
+    ORIGINAL_TEXT VARCHAR(16777216) COMMENT 'Original comment text',
+    PLATFORM VARCHAR(16777216) COMMENT 'Musora platform/brand',
+    COMMENT_TIMESTAMP TIMESTAMP_NTZ(9) COMMENT 'When comment was posted',
+    AUTHOR_NAME VARCHAR(16777216) COMMENT 'Commenter name',
+    AUTHOR_ID VARCHAR(16777216) COMMENT 'User ID',
+
+    -- Parent comment information
+    PARENT_COMMENT_ID VARCHAR(16777216) COMMENT 'ID of parent comment if this is a reply',
+    PARENT_COMMENT_TEXT VARCHAR(16777216) COMMENT 'Text of parent comment for context',
+
+    -- Content references
+    CONTENT_SK NUMBER(38,0) COMMENT 'Generated surrogate key for content',
+    CONTENT_ID VARCHAR(16777216) COMMENT 'Content ID',
+    CONTENT_DESCRIPTION VARCHAR(16777216) COMMENT 'Content profile/description',
+
+    -- Channel references
+    CHANNEL_SK NUMBER(38,0) COMMENT 'Generated surrogate key for channel',
+    CHANNEL_NAME VARCHAR(16777216) COMMENT 'Brand/channel name',
+    CHANNEL_DISPLAY_NAME VARCHAR(16777216) COMMENT 'Channel display name',
+
+    -- Musora-specific fields
+    PERMALINK_URL VARCHAR(16777216) COMMENT 'Web URL path of the content',
+    THUMBNAIL_URL VARCHAR(16777216) COMMENT 'Thumbnail URL of the content',
+
+    -- Language detection features
+    DETECTED_LANGUAGE VARCHAR(100) COMMENT 'Detected language name (e.g., English, Spanish)',
+    LANGUAGE_CODE VARCHAR(10) COMMENT 'ISO 639-1 language code (e.g., en, es)',
+    IS_ENGLISH BOOLEAN COMMENT 'True if comment is in English',
+    LANGUAGE_CONFIDENCE VARCHAR(20) COMMENT 'Confidence level: high, medium, low',
+    DETECTION_METHOD VARCHAR(50) COMMENT 'Method used: library, llm, or default',
+    HAS_TEXT BOOLEAN COMMENT 'True if comment has textual content (not just emojis)',
+
+    -- Translation features
+    TRANSLATED_TEXT VARCHAR(16777216) COMMENT 'English translation (or original if already English)',
+    TRANSLATION_PERFORMED BOOLEAN COMMENT 'True if translation was performed',
+    TRANSLATION_CONFIDENCE VARCHAR(20) COMMENT 'Translation confidence level',
+    TRANSLATION_NOTES VARCHAR(16777216) COMMENT 'Notes about translation',
+
+    -- Sentiment analysis features
+    SENTIMENT_POLARITY VARCHAR(20) COMMENT 'Sentiment: very_positive, positive, neutral, negative, very_negative',
+    INTENT VARCHAR(500) COMMENT 'Multi-label intents (comma-separated): praise, question, request, feedback_negative, suggestion, humor_sarcasm, off_topic, spam_selfpromo',
+    REQUIRES_REPLY BOOLEAN COMMENT 'True if comment requires a response (genuine questions/requests only)',
+    SENTIMENT_CONFIDENCE VARCHAR(20) COMMENT 'Sentiment analysis confidence: high, medium, low',
+    ANALYSIS_NOTES VARCHAR(16777216) COMMENT 'Concise notes with key topics/highlights for summarization',
+
+    -- Processing metadata
+    PROCESSING_SUCCESS BOOLEAN COMMENT 'True if processing completed successfully',
+    PROCESSING_ERRORS VARCHAR(16777216) COMMENT 'Any errors encountered during processing',
+    PROCESSED_AT TIMESTAMP_NTZ(9) COMMENT 'When this record was processed',
+    WORKFLOW_VERSION VARCHAR(20) COMMENT 'Version of the processing workflow',
+
+    -- Audit fields
+    CREATED_AT TIMESTAMP_NTZ(9) DEFAULT CURRENT_TIMESTAMP() COMMENT 'Record creation time',
+    UPDATED_AT TIMESTAMP_NTZ(9) DEFAULT CURRENT_TIMESTAMP() COMMENT 'Record update time'
+)
+COMMENT='ML Features table for language detection, translation, and sentiment analysis results from Musora internal app comments';
+
+-- Create indexes for common queries
+-- Note: Snowflake automatically optimizes queries, but we can define clustering keys
+ALTER TABLE MUSORA_COMMENT_SENTIMENT_FEATURES CLUSTER BY (COMMENT_TIMESTAMP, CHANNEL_NAME);
+
+-- Create view for Musora comments requiring reply
+CREATE OR REPLACE VIEW VW_MUSORA_COMMENTS_REQUIRING_REPLY AS
+SELECT
+    COMMENT_SK,
+    COMMENT_ID,
+    ORIGINAL_TEXT,
+    TRANSLATED_TEXT,
+    PARENT_COMMENT_ID,
+    PARENT_COMMENT_TEXT,
+    INTENT,
+    SENTIMENT_POLARITY,
+    SENTIMENT_CONFIDENCE,
+    CHANNEL_NAME,
+    AUTHOR_ID,
+    COMMENT_TIMESTAMP,
+    PLATFORM,
+    CONTENT_DESCRIPTION,
+    PERMALINK_URL,
+    THUMBNAIL_URL
+FROM MUSORA_COMMENT_SENTIMENT_FEATURES
+WHERE REQUIRES_REPLY = TRUE
+    AND PROCESSING_SUCCESS = TRUE
+ORDER BY COMMENT_TIMESTAMP DESC;
+
+-- Create view for Musora sentiment distribution
+CREATE OR REPLACE VIEW VW_MUSORA_SENTIMENT_DISTRIBUTION AS
+SELECT
+    CHANNEL_NAME,
+    SENTIMENT_POLARITY,
+    INTENT,
+    COUNT(*) AS COMMENT_COUNT,
+    COUNT(CASE WHEN REQUIRES_REPLY THEN 1 END) AS REPLIES_NEEDED,
+    COUNT(CASE WHEN PARENT_COMMENT_ID IS NOT NULL THEN 1 END) AS REPLY_COMMENTS,
+    AVG(CASE WHEN SENTIMENT_CONFIDENCE = 'high' THEN 3
+             WHEN SENTIMENT_CONFIDENCE = 'medium' THEN 2
+             WHEN SENTIMENT_CONFIDENCE = 'low' THEN 1
+             ELSE 0 END) AS AVG_CONFIDENCE_SCORE,
+    MAX(PROCESSED_AT) AS LAST_PROCESSED
+FROM MUSORA_COMMENT_SENTIMENT_FEATURES
+WHERE PROCESSING_SUCCESS = TRUE
+GROUP BY CHANNEL_NAME, SENTIMENT_POLARITY, INTENT
+ORDER BY CHANNEL_NAME, COMMENT_COUNT DESC;
+
+-- Create view for non-English Musora comments
+CREATE OR REPLACE VIEW VW_MUSORA_NON_ENGLISH_COMMENTS AS
+SELECT
+    COMMENT_SK,
+    COMMENT_ID,
+    ORIGINAL_TEXT,
+    DETECTED_LANGUAGE,
+    LANGUAGE_CODE,
+    TRANSLATED_TEXT,
+    TRANSLATION_CONFIDENCE,
+    SENTIMENT_POLARITY,
+    INTENT,
+    CHANNEL_NAME,
+    COMMENT_TIMESTAMP,
+    PLATFORM,
+    PERMALINK_URL
+FROM MUSORA_COMMENT_SENTIMENT_FEATURES
+WHERE IS_ENGLISH = FALSE
+    AND PROCESSING_SUCCESS = TRUE
+ORDER BY COMMENT_TIMESTAMP DESC;