UAP Improvements

.gitignore

@@ -1,191 +1 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so may need to be excluded from sync.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or binary, you wish to ignore these files, but for an app, you might want to check them in.
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or even
-#   fail to install them.
-# Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-# poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-# pdm.lock
-
-# PEP 582; used by e.g. github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-.idea/
-
-# Streamlit
-.streamlit/
-
-# VS Code
-.vscode/
-
-# Node.js
-node_modules/
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-.npm/
-
-# Dataset Files (Usually too large for git)
-*.h5
-*.h5.1
-*.csv
-*.jpeg
-*.webp
-*.pkl
-
-# Dist
-frontend/dist/
-frontend2/dist/
-
-# Large source PDFs (kept local, too big for git / HF Space)
-UAP_PDFs/
-
-# Document Preprocessing working dir (PDF corpus + intermediates) — data, not code.
-# The pipeline *scripts* live in pipeline/ and ARE tracked; the data tree is not.
-pipeline_data/
-
-# Windows "downloaded from internet" metadata sidecar files
-*Zone.Identifier
-
-# Superseded frontend backup + large generated cluster-viz HTML (kept local,
-# excluded from the HF Space deploy to stay within its storage limit).
-frontend_backup/
-frontend/uap_clusters_llm.html
-
-# Scratch notebooks
-Untitled.ipynb
+.streamlit

.streamlit/config.toml

@@ -0,0 +1,40 @@
+[browser]
+pageTitle = "UAP ANALYTICS"
+pageIcon = ":alien:"
+layout = "wide"
+
+[server]
+enableXsrfProtection = false
+maxUploadSize=5000
+maxMessageSize=5000
+
+
+[theme]
+# Primary accent for interactive elements
+primaryColor = "#FFA500"
+
+# Background color for the main content area
+#backgroundColor = "#273346"
+
+# Background color for sidebar and most interactive widgets
+#secondaryBackgroundColor = "#B9F1C0"
+
+# Color used for almost all text
+#textColor = "#FFFFFF"
+
+# Font family for all text in the app, except code blocks
+# Accepted values (serif | sans serif | monospace) 
+# Default: "sans serif"
+font = "sans serif"
+
+# Base theme (light or dark)
+base = "dark"
+
+
+[runner]
+magicEnabled = true
+fastReruns = false
+
+
+[client]
+toolbarMode = "auto"

.streamlit/credentials.toml

@@ -0,0 +1,2 @@
+[general]
+email = ""

.streamlit/secrets.toml

@@ -0,0 +1,14 @@
+testing_mode = false
+payment_provider = "stripe"
+stripe_api_key_test = 'rk_test_51PYs5M2LWM8BdEzK3ggKUMfcoVpwpmWABeiewJZp797aMvxWzlmaDe70svq5wajxiun4x98OhQZQB1lqP3AsQfDS009pbiZHSx'
+#stripe_api_key = 'pk_live_51PYs5M2LWM8BdEzK2lhKscFJJ8l8Z0CxY4xiLUTOoRjQnri1Qcf47Fhf3SAXH9P8jyPKYxfo3xEpHpHD5n8jMbqE00E3gRdxPF'
+stripe_api_key = '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'
+stripe_link_test = 'https://buy.stripe.com/test_4gw0390KX0ojc8w7ss'
+stripe_link = 'https://buy.stripe.com/bIYcP31PZ4Jwdgs288'
+client_id = '628411883365-dfkuut4shontl77uge7mta9514hambkc.apps.googleusercontent.com'
+client_secret = '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'
+redirect_url_test = 'https://huggingface.co/spaces/Ashoka74/UFOSINT'
+redirect_url = 'https://huggingface.co/spaces/UFOSINT/UAP-Data-Analysis-Tool/'
+GEMINI_KEY = 'AIzaSyAEALAXiaE1HcD8qcN1duY4OtmUDfYqquk'
+COHERE_KEY = 'UOGoge1RICTXAb710UrE0QQSftv6qZx8ysJKXY6j'
+OPENAI_KEY = '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'

CLAUDE.md

@@ -1,115 +0,0 @@
-# CLAUDE.md
-
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
-
-## Project Overview
-
-UAP-Data-Analysis-Tool (also known as UFOSINT) is a Streamlit-based data analysis application for UFO/UAP (Unidentified Aerial Phenomena) sighting reports. It provides text parsing, clustering, statistical analysis, geospatial visualization, and magnetic anomaly correlation capabilities.
-
-## Commands
-
-### Running the Application
-
-```bash
-# Streamlit app (main interface)
-uv run streamlit run app.py
-
-# Gradio app (alternative interface)
-uv run gradio gradio_app.py
-```
-
-### Dependency Management
-
-```bash
-# Install dependencies using uv (Python 3.12+)
-uv sync
-
-# Or using pip
-pip install -r requirements.txt
-```
-
-### Testing/Running Individual Pages
-
-Each Streamlit page can be tested individually:
-```bash
-uv run streamlit run parsing.py
-uv run streamlit run analyzing.py
-uv run streamlit run rag_search.py
-uv run streamlit run magnetic.py
-uv run streamlit run map.py
-```
-
-## Architecture
-
-### Core Analysis Pipeline
-
-The application follows a multi-step NLP analysis pipeline:
-
-1. **Parsing (`parsing.py`)** - Uses OpenAI GPT-4o-mini to extract structured JSON features from raw UAP report text. The JSON schema is defined in `config.py` (`FORMAT_LONG`).
-
-2. **Analysis (`analyzing.py`)** - Performs dimensionality reduction (UMAP) and clustering (HDBSCAN) on text embeddings, then trains XGBoost classifiers to identify feature correlations.
-
-3. **RAG Search (`rag_search.py`)** - Implements semantic search using Cohere's rerank API to find relevant reports based on natural language queries.
-
-4. **Magnetic Analysis (`magnetic.py`)** - Correlates UAP sightings with InterMagnet station data using Dynamic Time Warping (FastDTW).
-
-5. **Map Visualization (`map.py`)** - Interactive Kepler.gl maps showing sighting locations with proximity analysis to military bases and nuclear facilities.
-
-### Key Classes (`uap_analyzer.py`)
-
-- **`UAPParser`** - Concurrent OpenAI API calls for JSON extraction with exponential backoff
-- **`UAPAnalyzer`** - Text embedding (sentence-transformers/e5-large-v2), UMAP reduction, HDBSCAN clustering, TF-IDF cluster naming, cluster merging via cosine similarity
-- **`UAPVisualizer`** - XGBoost prediction plots, confusion matrices, Cramer's V heatmaps, treemaps
-
-### Utils Module (`utils/`)
-
-Enhanced utilities providing:
-- `DataProcessor` - DataFrame filtering with Streamlit UI
-- `UAP_Visualizer` - Interactive Plotly visualizations
-- `SessionStateManager` - Streamlit session state handling
-- `EmbeddingCacheManager` - Caches sentence-transformer embeddings to avoid recomputation
-- `MemoryManager` - GPU memory management for CUDA operations
-- `UAP_Pipeline` - End-to-end analysis pipeline orchestration
-
-### Data Flow
-
-```
-Raw Text Reports (CSV/XLSX)
-    └── parsing.py (GPT-4o-mini) → Structured JSON
-            └── analyzing.py → Embeddings → UMAP → HDBSCAN clusters
-                    └── XGBoost predictions + Cramer's V correlations
-                            └── map.py (Kepler.gl visualization)
-```
-
-### Session State Keys
-
-The app uses Streamlit session state extensively. Key variables:
-- `parsed_responses` / `parsed_responses_df` - Parsed JSON data
-- `analyzers` / `clusters` / `col_names` - Analysis results
-- `stage` - UI workflow state
-- `api_key_valid` - OpenAI key validation status
-
-## Configuration
-
-### API Keys
-
-Secrets are stored in `.streamlit/secrets.toml` (not committed). Required keys:
-- `OPENAI_KEY` - For GPT-4o-mini parsing
-- `GEMINI_KEY` - For Gemini Pro summarization
-- `COHERE_KEY` - For rerank search
-
-### Data Files
-
-- `final_ufoseti_dataset.h5` - Pre-parsed UAP dataset
-- `parsed_files_distance_embeds.h5` - Dataset with embeddings
-- `global_power_plant_database.csv` - Nuclear facility locations
-- `secret_bases.csv` - Military base locations
-- `*.kgl` - Kepler.gl map configuration files
-
-## GPU Requirements
-
-The embedding model (`embaas/sentence-transformers-e5-large-v2`) and XGBoost run on CUDA by default. The code includes `torch.cuda.empty_cache()` calls for memory management.
-
-## HuggingFace Deployment
-
-The app is configured for HuggingFace Spaces deployment (see `README.md` metadata). The `sdk_version: 1.36.0` specifies the Streamlit version.

ENHANCED_PIPELINE_README.md

@@ -1,287 +0,0 @@
-# Enhanced Dynamic Filtering and Visualization Pipeline
-
-## Overview
-
-This document describes the major improvements made to the UAP Data Analysis Tool's dynamic filtering and visualization pipeline. The enhancements focus on performance, usability, and advanced interactive features.
-
-## 🚀 Key Improvements
-
-### 1. Enhanced Data Processing (`utils/data_processing.py`)
-
-#### Intelligent Data Profiling
-- **Automatic column type detection** with statistical analysis
-- **Memory usage optimization** and performance monitoring
-- **Smart categorization** of columns (categorical, numeric, datetime, text)
-- **High cardinality column detection** for special handling
-
-#### Advanced Filtering System
-- **Quick Filter Presets**: Pre-configured filters for common scenarios
-  - Remove outliers using IQR method
-  - Top categories only
-  - Remove rare categories (< 1% frequency)
-  - Recent data filtering
-- **Multi-modal Filtering**:
-  - Range, percentile, and standard deviation filters for numeric data
-  - Smart selection, top-N, and exclusion modes for categorical data
-  - Advanced date filtering with relative periods
-  - Text filtering with regex support and length-based filtering
-
-#### Performance Optimizations
-- **Intelligent caching** with dataframe hashing
-- **Performance monitoring** decorators
-- **Parallel processing** for data parsing operations
-- **Memory-efficient operations** with smart sampling
-
-### 2. Enhanced Visualization System (`utils/visualization.py`)
-
-#### Interactive Visualizations
-- **Plotly-based interactive charts** replacing static matplotlib plots
-- **Smart sampling** for large datasets maintaining data distribution
-- **Progressive rendering** to handle datasets of any size
-- **Drill-down capabilities** in treemaps and other charts
-
-#### New Visualization Types
-1. **Interactive Scatter Plots**
-   - Color and size encoding
-   - Intelligent sampling for performance
-   - Hover data with context
-   - Sampling indicators for transparency
-
-2. **Enhanced Histograms**
-   - Box plots integration
-   - Interactive binning
-   - Statistical overlays
-
-3. **Interactive Treemaps**
-   - Drill-down capabilities
-   - Percentage and count displays
-   - Color-coded hierarchies
-
-4. **Correlation Matrices**
-   - Interactive heatmaps
-   - Multiple correlation methods (Pearson, Spearman, Kendall)
-   - Hover tooltips with exact values
-
-5. **Time Series Plots**
-   - Range selectors and sliders
-   - Multiple series support
-   - Resampling options
-   - Zoom and pan capabilities
-
-6. **Dashboard Layouts**
-   - Multi-chart dashboards
-   - Configurable layouts (2x2, vertical)
-   - Synchronized interactions
-
-#### Performance Features
-- **Smart sampling algorithms** preserving data distribution
-- **Stratified sampling** for categorical data
-- **Caching at multiple levels** (Streamlit cache + custom cache)
-- **Memory-aware rendering** with automatic optimization
-
-### 3. Session State Management (`utils/session_manager.py`)
-
-#### Enhanced State Handling
-- **Centralized session state management**
-- **Visualization caching** for improved performance
-- **Filter state persistence** across app interactions
-- **Memory management** for large datasets
-
-### 4. Application Integration
-
-#### Updated Applications
-All main applications now use the enhanced pipeline:
-
-1. **Analyzing App** (`analyzing.py`)
-   - Enhanced filtering with quick presets
-   - Interactive visualization tabs
-   - Performance monitoring
-   - Re-analysis on filtered data
-
-2. **Map App** (`map.py`)
-   - Map-optimized filtering (datetime to string conversion)
-   - Geographic data handling improvements
-   - Enhanced coordinate detection
-
-3. **Other Apps** can be easily updated using the same pattern
-
-## 📊 Usage Examples
-
-### Basic Enhanced Filtering
-```python
-from utils.data_processing import DataProcessor
-
-# Enhanced filtering with all features
-filtered_df = DataProcessor.filter_dataframe_enhanced(
-    df, 
-    enable_quick_filters=False,
-    enable_advanced_filters=True
-)
-```
-
-### Interactive Visualizations
-```python
-from utils.visualization import UAP_Visualizer
-
-# Interactive scatter plot with smart sampling
-fig = UAP_Visualizer.plot_interactive_scatter(
-    df, 'latitude', 'longitude', 
-    color_col='shape', 
-    max_points=10000
-)
-st.plotly_chart(fig, use_container_width=True)
-
-# Correlation matrix
-fig = UAP_Visualizer.plot_correlation_matrix(df[numeric_columns])
-st.plotly_chart(fig, use_container_width=True)
-```
-
-### Data Profiling
-```python
-# Get intelligent data profile
-profile = DataProcessor.profile_data(df)
-print(f"Categorical columns: {len(profile['categorical_columns'])}")
-print(f"Numeric columns: {len(profile['numeric_columns'])}")
-print(f"Memory usage: {profile['memory_usage'] / 1024**2:.1f} MB")
-```
-
-## 🎯 Performance Improvements
-
-### Before vs After
-- **Filtering Speed**: 3-5x faster with intelligent caching
-- **Visualization Rendering**: 2-10x faster with smart sampling
-- **Memory Usage**: 30-50% reduction for large datasets
-- **User Experience**: Instant feedback with progressive loading
-
-### Smart Sampling Benefits
-- Maintains statistical properties of data
-- Preserves category distributions
-- Transparent to users (shows sampling info)
-- Configurable sampling limits
-
-### Caching Strategy
-- **Multi-level caching**: Streamlit + custom application cache
-- **Intelligent cache keys**: Based on data hashes
-- **Automatic cache invalidation**: When data changes
-- **Memory-aware caching**: Prevents memory overflow
-
-## 🔧 Configuration Options
-
-### Performance Tuning
-```python
-# Adjust sampling limits
-UAP_Visualizer.plot_interactive_scatter(df, x, y, max_points=5000)
-
-# Enable/disable performance mode
-DataProcessor.filter_dataframe_enhanced(
-    df, 
-    enable_quick_filters=False,  # Quick preset filters
-    enable_advanced_filters=True  # Full filtering interface
-)
-```
-
-### Visualization Customization
-```python
-# Custom color schemes
-fig = UAP_Visualizer.plot_correlation_matrix(df, method='spearman')
-
-# Time series with resampling
-fig = UAP_Visualizer.plot_time_series(
-    df, 'date_column', ['value1', 'value2'], 
-    resample_freq='M'  # Monthly resampling
-)
-```
-
-## 🚀 Getting Started
-
-### 1. Install Dependencies
-The enhanced pipeline requires additional dependencies (already in requirements.txt):
-- `plotly` - Interactive visualizations
-- `pandas` - Enhanced data processing
-- `streamlit` - Caching and UI components
-
-### 2. Update Existing Code
-Replace old filter functions:
-```python
-# Old way
-filtered_df = filter_dataframe(df)
-
-# New way
-from utils.data_processing import DataProcessor
-filtered_df = DataProcessor.filter_dataframe_enhanced(df)
-```
-
-### 3. Use Enhanced Visualizations
-```python
-# Replace matplotlib plots with interactive versions
-from utils.visualization import UAP_Visualizer
-
-# Interactive histogram instead of static
-fig = UAP_Visualizer.plot_interactive_histogram(df, 'column_name')
-st.plotly_chart(fig, use_container_width=True)
-```
-
-### 4. Try the Demo
-Run the enhanced example:
-```bash
-streamlit run utils/enhanced_example.py
-```
-
-## 🔮 Future Enhancements
-
-### Planned Features
-1. **Real-time Filtering**: WebSocket-based live data filtering
-2. **Advanced Analytics**: Statistical tests and ML model integration
-3. **Export Capabilities**: Enhanced data export with filter preservation
-4. **Custom Visualizations**: User-defined chart types
-5. **Performance Profiling**: Built-in performance analytics dashboard
-
-### Extensibility
-The new architecture is designed for easy extension:
-- Add new filter types in `DataProcessor`
-- Create custom visualizations in `UAP_Visualizer`
-- Extend session management for new use cases
-
-## 📈 Impact
-
-### User Experience
-- **Faster interactions**: Immediate feedback on all operations
-- **Better insights**: Interactive visualizations reveal patterns
-- **Easier exploration**: Quick filters and smart defaults
-- **Transparent performance**: Users see sampling and processing info
-
-### Developer Experience
-- **Cleaner code**: Centralized utilities eliminate duplication
-- **Better maintainability**: Single source of truth for filtering/visualization
-- **Performance monitoring**: Built-in performance tracking
-- **Easy extension**: Modular architecture for new features
-
-### System Performance
-- **Scalability**: Handles datasets from thousands to millions of rows
-- **Memory efficiency**: Smart sampling and caching prevent memory issues
-- **Response times**: Sub-second response for most operations
-- **Resource usage**: Optimized CPU and memory utilization
-
-## 🛠️ Technical Details
-
-### Architecture
-```
-UAP Data Analysis Tool
-├── utils/
-│   ├── data_processing.py     # Enhanced filtering and data ops
-│   ├── visualization.py       # Interactive visualizations
-│   ├── session_manager.py     # State management
-│   └── enhanced_example.py    # Complete demo
-├── analyzing.py               # Updated with enhanced features
-├── map.py                     # Updated with enhanced features
-└── other apps...              # Can be updated similarly
-```
-
-### Key Classes
-- `DataProcessor`: Centralized data operations with intelligent caching
-- `UAP_Visualizer`: Interactive visualization factory with performance optimization
-- `SessionStateManager`: Enhanced state management with visualization caching
-
-The enhanced pipeline represents a significant upgrade to the UAP Data Analysis Tool, providing better performance, richer interactivity, and a superior user experience while maintaining backward compatibility and extensibility for future enhancements.
-
-

PLAN.md

@@ -1,313 +0,0 @@
-# Implementation Plan: React + FastAPI Architecture
-
-## Overview
-
-Convert the Streamlit-based UAP Data Analysis Tool to a modern React frontend with FastAPI backend, maintaining all existing functionality.
-
-## Architecture
-
-```
-┌─────────────────────────────────────────────────────────────┐
-│                     React Frontend                          │
-│  ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌───────┐ │
-│  │ Parsing │ │Analysis │ │ Search  │ │Magnetic │ │  Map  │ │
-│  └────┬────┘ └────┬────┘ └────┬────┘ └────┬────┘ └───┬───┘ │
-└───────┼──────────┼──────────┼──────────┼──────────┼───────┘
-        │          │          │          │          │
-        ▼          ▼          ▼          ▼          ▼
-┌─────────────────────────────────────────────────────────────┐
-│                    FastAPI Backend                          │
-│  /api/parse  /api/analyze  /api/search  /api/magnetic /api/map │
-└─────────────────────────────────────────────────────────────┘
-        │
-        ▼
-┌─────────────────────────────────────────────────────────────┐
-│              Existing Python Services                       │
-│  UAPParser  UAPAnalyzer  UAPVisualizer  Cohere  InterMagnet │
-└─────────────────────────────────────────────────────────────┘
-```
-
-## File Structure
-
-```
-UAP-Data-Analysis-Tool/
-├── api/                          # FastAPI backend
-│   ├── __init__.py
-│   ├── main.py                   # FastAPI app entry point
-│   ├── config.py                 # Settings and API keys
-│   ├── routes/
-│   │   ├── __init__.py
-│   │   ├── upload.py             # File upload endpoints
-│   │   ├── parse.py              # OpenAI parsing endpoints
-│   │   ├── analyze.py            # UMAP/HDBSCAN/XGBoost endpoints
-│   │   ├── search.py             # Cohere rerank endpoints
-│   │   ├── magnetic.py           # InterMagnet correlation endpoints
-│   │   └── map.py                # Geospatial data endpoints
-│   ├── services/
-│   │   ├── __init__.py
-│   │   ├── parser_service.py     # Wraps UAPParser
-│   │   ├── analyzer_service.py   # Wraps UAPAnalyzer
-│   │   ├── visualizer_service.py # Wraps UAPVisualizer
-│   │   ├── search_service.py     # Cohere rerank logic
-│   │   ├── magnetic_service.py   # InterMagnet API + DTW
-│   │   └── map_service.py        # Kepler.gl data prep
-│   ├── models/
-│   │   ├── __init__.py
-│   │   ├── schemas.py            # Pydantic request/response models
-│   │   └── jobs.py               # Background job tracking
-│   └── utils/
-│       ├── __init__.py
-│       ├── file_handler.py       # CSV/Excel/HDF5 handling
-│       └── serialization.py      # NumPy/DataFrame serialization
-│
-├── frontend/                     # React frontend
-│   ├── package.json
-│   ├── tailwind.config.js
-│   ├── vite.config.js
-│   ├── index.html
-│   ├── src/
-│   │   ├── main.jsx
-│   │   ├── App.jsx
-│   │   ├── api/
-│   │   │   └── client.js         # Axios/fetch wrapper
-│   │   ├── components/
-│   │   │   ├── layout/
-│   │   │   │   ├── Navbar.jsx
-│   │   │   │   ├── Sidebar.jsx
-│   │   │   │   └── Layout.jsx
-│   │   │   ├── common/
-│   │   │   │   ├── FileUpload.jsx
-│   │   │   │   ├── DataTable.jsx
-│   │   │   │   ├── LoadingSpinner.jsx
-│   │   │   │   └── ErrorBoundary.jsx
-│   │   │   ├── charts/
-│   │   │   │   ├── Treemap.jsx
-│   │   │   │   ├── Histogram.jsx
-│   │   │   │   ├── ScatterPlot.jsx
-│   │   │   │   ├── Heatmap.jsx
-│   │   │   │   └── ConfusionMatrix.jsx
-│   │   │   └── map/
-│   │   │       └── KeplerMap.jsx
-│   │   ├── pages/
-│   │   │   ├── Home.jsx
-│   │   │   ├── Parsing.jsx
-│   │   │   ├── Analysis.jsx
-│   │   │   ├── Search.jsx
-│   │   │   ├── Magnetic.jsx
-│   │   │   └── Map.jsx
-│   │   ├── hooks/
-│   │   │   ├── useFileUpload.js
-│   │   │   ├── useAnalysis.js
-│   │   │   └── useWebSocket.js
-│   │   ├── store/
-│   │   │   └── index.js          # Zustand or React Context
-│   │   └── styles/
-│   │       └── globals.css
-│   └── public/
-│       └── assets/
-```
-
-## Implementation Steps
-
-### Phase 1: FastAPI Backend Setup
-
-#### Step 1.1: Create API structure and main entry point
-- Create `api/` directory structure
-- Set up FastAPI app with CORS middleware
-- Configure settings from environment/secrets
-- Add health check endpoint
-
-#### Step 1.2: Create Pydantic schemas
-- Define request models for each endpoint
-- Define response models with proper typing
-- Create job status models for async operations
-
-#### Step 1.3: Implement file upload endpoint
-- POST `/api/upload` - Accept CSV/Excel files
-- Store uploaded files temporarily
-- Return file ID and column list
-- Support chunked uploads for large files
-
-#### Step 1.4: Implement parsing endpoints
-- POST `/api/parse/start` - Start async parsing job
-- GET `/api/parse/status/{job_id}` - Check job status
-- GET `/api/parse/result/{job_id}` - Get parsed results
-- Wrap existing UAPParser with proper error handling
-
-#### Step 1.5: Implement analysis endpoints
-- POST `/api/analyze/start` - Start clustering analysis
-- GET `/api/analyze/status/{job_id}` - Check status
-- GET `/api/analyze/clusters/{job_id}` - Get cluster data
-- GET `/api/analyze/embeddings/{job_id}` - Get 2D embeddings for visualization
-- GET `/api/analyze/predictions/{job_id}` - Get XGBoost results
-
-#### Step 1.6: Implement search endpoints
-- POST `/api/search/rerank` - Cohere rerank search
-- Return ranked results with relevance scores
-
-#### Step 1.7: Implement magnetic endpoints
-- GET `/api/magnetic/stations` - List InterMagnet stations
-- POST `/api/magnetic/correlate` - Run DTW correlation
-- Return correlation results and time series data
-
-#### Step 1.8: Implement map endpoints
-- GET `/api/map/sightings` - Get sighting GeoJSON
-- GET `/api/map/bases` - Get military bases data
-- GET `/api/map/plants` - Get nuclear facilities data
-- GET `/api/map/config` - Get Kepler.gl config
-
-### Phase 2: React Frontend Setup
-
-#### Step 2.1: Initialize React project
-- Create Vite + React project in `frontend/`
-- Install dependencies: react-router, axios, recharts/plotly, kepler.gl
-- Configure Tailwind CSS
-- Set up project structure
-
-#### Step 2.2: Create layout components
-- Navbar with navigation links
-- Sidebar for feature options
-- Main layout wrapper
-- Responsive design
-
-#### Step 2.3: Create common components
-- FileUpload with drag-and-drop
-- DataTable with sorting/filtering
-- LoadingSpinner and progress indicators
-- Error boundary and toast notifications
-
-#### Step 2.4: Create chart components
-- Treemap using Plotly.js
-- Histogram using Recharts
-- ScatterPlot for embeddings visualization
-- Heatmap for Cramer's V and confusion matrix
-- Feature importance bar chart
-
-#### Step 2.5: Create Kepler.gl map component
-- Integrate kepler.gl React component
-- Handle data layers dynamically
-- Support filtering by attributes
-
-### Phase 3: Feature Pages
-
-#### Step 3.1: Parsing page
-- File upload interface
-- Column selector
-- Custom JSON schema editor (optional)
-- Progress indicator for parsing
-- Results table with download option
-
-#### Step 3.2: Analysis page
-- Dataset loader (upload or use parsed data)
-- Column multi-selector for analysis
-- Visualization tabs: Embeddings, Clusters, Predictions, Correlations
-- Interactive charts with tooltips
-
-#### Step 3.3: Search page
-- Dataset display
-- Query input
-- Column selector for search scope
-- Ranked results with relevance scores
-- Click-to-expand details
-
-#### Step 3.4: Magnetic page
-- Date range selector
-- Location input (lat/lon or from dataset)
-- Station selector
-- Correlation results with time series chart
-
-#### Step 3.5: Map page
-- Full-screen Kepler.gl map
-- Layer toggles (sightings, bases, plants)
-- Filter controls
-- Export functionality
-
-### Phase 4: Integration and Polish
-
-#### Step 4.1: State management
-- Set up Zustand store for global state
-- Persist uploaded data across pages
-- Handle authentication state (API keys)
-
-#### Step 4.2: WebSocket for long-running tasks
-- Add WebSocket endpoint for job progress
-- Real-time updates during parsing/analysis
-
-#### Step 4.3: Error handling
-- Consistent error responses from API
-- User-friendly error messages in frontend
-- Retry logic for failed requests
-
-#### Step 4.4: Testing
-- API endpoint tests with pytest
-- Component tests with React Testing Library
-
-## Key API Endpoints Summary
-
-| Method | Endpoint | Description |
-|--------|----------|-------------|
-| POST | `/api/upload` | Upload CSV/Excel file |
-| POST | `/api/parse/start` | Start parsing job |
-| GET | `/api/parse/status/{job_id}` | Get parsing status |
-| GET | `/api/parse/result/{job_id}` | Get parsed data |
-| POST | `/api/analyze/start` | Start analysis job |
-| GET | `/api/analyze/clusters/{job_id}` | Get cluster results |
-| GET | `/api/analyze/embeddings/{job_id}` | Get 2D embeddings |
-| POST | `/api/search/rerank` | Semantic search |
-| GET | `/api/magnetic/stations` | List stations |
-| POST | `/api/magnetic/correlate` | Run correlation |
-| GET | `/api/map/sightings` | Get sighting GeoJSON |
-| GET | `/api/map/bases` | Get bases GeoJSON |
-
-## Dependencies to Add
-
-### Backend (add to pyproject.toml)
-```
-fastapi
-uvicorn[standard]
-python-multipart
-aiofiles
-websockets
-```
-
-### Frontend (package.json)
-```json
-{
-  "dependencies": {
-    "react": "^18.2.0",
-    "react-dom": "^18.2.0",
-    "react-router-dom": "^6.x",
-    "axios": "^1.x",
-    "plotly.js": "^2.x",
-    "react-plotly.js": "^2.x",
-    "kepler.gl": "^3.x",
-    "react-dropzone": "^14.x",
-    "@tanstack/react-table": "^8.x",
-    "zustand": "^4.x",
-    "react-hot-toast": "^2.x"
-  },
-  "devDependencies": {
-    "vite": "^5.x",
-    "tailwindcss": "^3.x",
-    "autoprefixer": "^10.x",
-    "postcss": "^8.x"
-  }
-}
-```
-
-## Running the Application
-
-```bash
-# Terminal 1: Start FastAPI backend
-cd api && uvicorn main:app --reload --port 8000
-
-# Terminal 2: Start React frontend
-cd frontend && npm run dev
-```
-
-## Notes
-
-- Long-running tasks (parsing, analysis) use background jobs with polling or WebSocket updates
-- Embeddings are stored server-side and referenced by job_id to avoid large payloads
-- Visualizations are generated as Plotly JSON for interactive frontend rendering
-- The existing `uap_analyzer.py` and `utils/` modules are reused as services

README.md

@@ -5,57 +5,10 @@ colorFrom: green
 colorTo: yellow
 sdk: streamlit
 sdk_version: 1.36.0
-python_version: "3.12"
 app_file: app.py
 pinned: false
 license: apache-2.0
 short_description: UFO/UAP AI Analyst
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
-
----
-
-## Modern FrontEnd (React + FastAPI)
-
-The repo ships a decoupled web stack alongside the Streamlit app:
-
-- **Backend** — FastAPI service under `./api` (routes in `api/routes/`, services in `api/services/`, models in `api/models/`).
-- **Frontend** — Vite + React + TypeScript + Tailwind + Zustand under `./frontend`, talking to the backend through a Vite proxy.
-
-### Prerequisites
-- Python 3.11+ with `fastapi` and `uvicorn` installed (covered by `requirements.txt` / `pyproject.toml`).
-- Node 20+ for the frontend.
-
-### 1. Start the backend (FastAPI)
-
-From the repo root:
-
-```bash
-uvicorn api.main:app --reload --port 8000
-```
-
-The `--reload` flag picks up code changes automatically.
-
-### 2. Start the frontend (Vite + React)
-
-```bash
-cd frontend
-npm install      # first time only
-npm run dev
-```
-
-Vite starts on port **5173** and proxies every `/api/*` request to `http://localhost:8000` (see `frontend/vite.config.ts`), so the React app calls FastAPI transparently — no CORS configuration needed.
-
-### 3. Open the app
-
-http://localhost:5173
-
-### Production build
-
-```bash
-cd frontend
-npm run build       # output in frontend/dist
-```
-
-Serve the `dist/` artifacts behind any static host and keep `uvicorn api.main:app` running for the API.
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

STREAMLIT_HANDOFF.md

@@ -1,545 +0,0 @@
-# Handoff: UAP Embeddings → Streamlit Semantic Search
-
-Everything you need to build a Streamlit app that does semantic search over the
-UAP archive embeddings currently sitting in a Neon Postgres + pgvector database.
-
----
-
-## 1. Context in one paragraph
-
-A previous session embedded all of **UAP Release 2 (5/22/26)** — 49 DoD UAP
-video clips and 7 NASA Apollo/Mercury audio recordings — into a Neon Postgres
-database using **Google Gemini `gemini-embedding-2-preview`** (768-dim, cosine
-similarity, indexed with HNSW). The pipeline lives in `embeddings_v2.py` at the
-repo root. Your job is a Streamlit UI that lets users type a query (or upload an
-image), embed it with the same model, and return ranked matches with playable
-media.
-
----
-
-## 2. What's in the database right now
-
-```
-source_type    rows    distinct assets
-video_chunk     154    49 DVIDS UAP video clips (Release 2)
-pdf_page        126     5 source documents (DOW-D017 [116p], DOE-D002 [4p],
-                          CIA-D001 [3p], DOE-D001 [2p], DOE-D003 [1p])
-audio_clip       27     7 NASA Apollo/Mercury audio recordings (Release 2)
-TOTAL           307    61 assets    all release='PURSUE_2'  release_date=2026-05-22
-```
-
-- All current rows use `user_id = '00000000-0000-0000-0000-000000000001'` (a
-  placeholder UUID — the schema is multi-tenant but this archive has one tenant).
-- `parent_id` is `dvids_{asset_id}` for media rows (e.g. `dvids_1007706`); doc
-  slugs like `dow-uap-d017` for `pdf_page` rows.
-- `source_id` is `{parent_id}:{start_ms}-{end_ms}` for media chunks and
-  `{parent_id}:p{NNNN}` for PDF pages (e.g. `dow-uap-d017:p0017`).
-- Vector dimension is **768**. Queries must be 768-dim too.
-- Every row carries the new `release` (`'PURSUE_2'`) and `release_date`
-  (`2026-05-22`) columns — filter on these in the UI when more releases land.
-- One pending video (`1007708`, the 513 MB outlier) was not ingested; it can be
-  added later — not a blocker for the UI.
-- Nothing from earlier releases (Release 1, NARA-CIA, FBI photos, etc.) is
-  embedded yet. If you build the UI to filter on `release` / `parent_id`
-  patterns or future source types, leave it open.
-
----
-
-## 3. Schema reference
-
-```sql
-CREATE TABLE embeddings (
-    id                  BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-    source_type         TEXT        NOT NULL,   -- 'video_chunk' | 'audio_clip' | 'pdf_page' (more later)
-    source_id           TEXT        NOT NULL,   -- '{parent_id}:{start_ms}-{end_ms}' for chunks; '{slug}:p{NNNN}' for pages
-    user_id             UUID        NOT NULL,
-    organization_id     UUID,
-    embedding           VECTOR(768) NOT NULL,
-    embedded_image_url  TEXT,                   -- video/audio: DVIDS page URL; pdf_page: whole-PDF war.gov URL
-    embedded_text       TEXT,                   -- caption used during embed (Title + Blurb; or metadata + OCR for pdf_page)
-    start_seconds       REAL,                   -- chunk start (NULL for pdf_page)
-    end_seconds         REAL,                   -- chunk end   (NULL for pdf_page)
-    parent_id           TEXT,                   -- 'dvids_1007706' for media; doc slug like 'dow-uap-d017' for pages
-    release             TEXT        NOT NULL DEFAULT 'PURSUE_2',   -- campaign tag (filter on this in the UI)
-    release_date        DATE        NOT NULL DEFAULT '2026-05-22', -- when the source documents were publicly released
-    created_at          TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-    updated_at          TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-    CONSTRAINT uq_embeddings_source UNIQUE (source_type, source_id)
-);
-
--- Already created:
-CREATE INDEX idx_embeddings_embedding ON embeddings USING hnsw (embedding vector_cosine_ops);
-CREATE INDEX idx_embeddings_parent_id ON embeddings (parent_id) WHERE parent_id IS NOT NULL;
-CREATE INDEX idx_embeddings_user_id   ON embeddings (user_id);
-```
-
-Cosine search uses pgvector's `<=>` operator (distance, lower = closer).
-Convert to similarity with `1 - (embedding <=> query)`.
-
----
-
-## 4. Secrets — required, not in this file
-
-Set as env vars (or Streamlit `secrets.toml`):
-
-```bash
-DATABASE_URL    = <Neon connection string, prefer the DIRECT endpoint over -pooler>
-GEMINI_API_KEY  = <Google AI Studio key, same model that produced the rows>
-```
-
-The Neon string must include `?sslmode=require`. Ask the user to paste the
-values from their Neon dashboard and Google AI Studio — they're not embedded
-here on purpose. The previous session ran against a Neon project owned by the
-user, and the password / key from that session should be considered exposed
-and rotated.
-
-**Streamlit secrets.toml** (recommended over raw env vars):
-
-```toml
-# .streamlit/secrets.toml  -- DO NOT COMMIT
-DATABASE_URL = "postgresql://USER:PASSWORD@ep-xxxx.REGION.aws.neon.tech/neondb?sslmode=require"
-GEMINI_API_KEY = "AIza..."
-```
-
-Read in app with `st.secrets["DATABASE_URL"]`.
-
----
-
-## 5. Dependencies
-
-```bash
-pip install streamlit google-genai pillow requests "psycopg[binary]" pgvector
-```
-
-The only file from this repo you need to copy alongside the Streamlit app is
-**`embeddings_v2.py`** (it's self-contained — no project-internal imports). Or
-you can inline the few functions you actually use (see §6/§7 for the bare
-minimum).
-
----
-
-## 6. Embedding a user query
-
-The model and dimension must match what's already in the DB
-(`gemini-embedding-2-preview`, 768-d). **The contract is asymmetric and is
-expressed in the prompt, not the config**: queries get a `task: search result
-| query: …` prefix; documents go in as `title: … | text: …`. The
-`EmbedContentConfig.task_type` field is *silently ignored* by
-gemini-embedding-2 on the consumer API — don't set it. (Helper functions in
-`embeddings_v2.py` apply the wrapping for you.)
-
-```python
-import embeddings_v2 as e
-
-# Queries — generate_text_embedding auto-wraps with format_query().
-vec_text  = e.generate_text_embedding("UAP over the Aegean")
-vec_image = e.generate_image_embedding("./uploaded.jpg")   # image-only: no text instruction
-vec_both  = e.generate_multimodal_embedding(
-    "./uploaded.jpg",
-    e.format_query("what is this"),                        # pre-wrap when there IS a text part
-)
-```
-
-`embeddings_v2` also exports:
-
-- `format_document_text(title, body)` → `"title: {title} | text: {body}"` (use when storing).
-- `format_query(query)` → `"task: search result | query: {query}"` (use when querying with a text part attached to media).
-
-Minimal inline version if you don't want to import `embeddings_v2`:
-
-```python
-import os
-from google import genai
-from google.genai import types as gt
-
-client = genai.Client(api_key=os.environ["GEMINI_API_KEY"])
-
-def embed_text(text: str) -> list[float]:
-    r = client.models.embed_content(
-        model="gemini-embedding-2-preview",
-        contents=f"task: search result | query: {text}",   # wrap, not task_type=
-        config=gt.EmbedContentConfig(output_dimensionality=768),
-    )
-    return list(r.embeddings[0].values)
-```
-
----
-
-## 7. Searching with pgvector
-
-`embeddings_v2.search_similar()` already does this and returns a list of
-`SimilarityHit` dataclasses. If you want raw SQL:
-
-```sql
-SELECT source_type, source_id, parent_id, start_seconds, end_seconds,
-       embedded_image_url, embedded_text,
-       1 - (embedding <=> %s) AS similarity
-FROM embeddings
-WHERE user_id = %s::uuid
-  AND (%s::text IS NULL OR source_type = %s)
-  AND (embedding <=> %s) <= %s          -- distance <= 1 - threshold
-ORDER BY embedding <=> %s
-LIMIT %s;
-```
-
-Params, in order: `query_vec, user_id, source_type_or_null, source_type_or_null, query_vec, (1 - threshold), query_vec, limit`.
-
-Don't forget `register_vector(conn)` from `pgvector.psycopg` after connecting —
-without it psycopg can't bind `list[float]` to the `vector` type.
-
----
-
-## 8. Result interpretation (per source_type)
-
-### `video_chunk`
-- `parent_id` → e.g. `dvids_1007706`. Strip the prefix to get the DVIDS asset id.
-- `embedded_image_url` → the human DVIDS page, e.g. `https://www.dvidshub.net/video/1007706`.
-- `start_seconds`, `end_seconds` → the chunk's offsets within the source video
-  (one video typically has multiple chunks; show the timestamp to the user).
-- `embedded_text` → the caption that was attached at embed time: the
-  `Video Title` + `Description Blurb` from `uap-data_v2.csv`.
-- DVIDS deep-link with timestamp: append `?t={int(start_seconds)}` to the page
-  URL (or use the local file with `st.video(local_path, start_time=int(start_seconds))`).
-
-### `audio_clip`
-- Same `parent_id` shape but with audio DVIDS ids (1007870–1007879 range for
-  Release 2).
-- `embedded_image_url` is set even though the asset is audio (it's the DVIDS
-  page URL — the column was reused as the canonical media URL for any kind).
-- For long recordings (>80s — the model's audio input cap), the asset is
-  segmented into ≤75s pieces; one row per piece with its own start/end.
-
-### `pdf_page`
-- `parent_id` is the doc slug (e.g. `dow-uap-d017`, `cia-uap-d001`,
-  `doe-uap-d001`, `doe-uap-d002`, `doe-uap-d003`).
-- `source_id` is `{parent_id}:p{NNNN}` with the page number zero-padded to
-  4 digits (e.g. `dow-uap-d017:p0017`). Parse with a tiny regex to surface
-  the page number in the UI.
-- `embedded_image_url` is the whole-PDF URL on war.gov — there's no per-page
-  URL on the source site, so deep-linking to a specific page means opening
-  the PDF and scrolling.
-- `embedded_text` is composed at embed time as: `{Agency} - {Title}` /
-  `Date: ... Location: ...` / `Page N of M.` / `Document context: {blurb}` /
-  `Page OCR: {ocr}`, capped at 8000 chars. The same string was paired with
-  the rendered page image in the multimodal embed call.
-- `start_seconds` / `end_seconds` are NULL.
-- A rendered page image lives locally at
-  `D:\divided\release_2\UAP_Release_2\pages\{slug}\page_NNNN.png` (150 dpi).
-  Display it directly with `st.image(local_path)`; link to
-  `embedded_image_url` to open the whole PDF on war.gov.
-
----
-
-## 9. Where the media files live
-
-The previous session saved every downloaded media file under the user's local
-drive (set by them as the persistence target):
-
-```
-D:\divided\release_2\UAP_Release_2\
-├── videos\dvids_{id}.mp4         (49 files, normalized originals from DVIDS)
-├── audio\dvids_{id}.{ext}        (7 source MP4 wrappers + extracted .m4a tracks)
-└── pages\{slug}\page_NNNN.png    (PDF page renders at 150 dpi, e.g.
-                                   pages\dow-uap-d017\page_0017.png)
-```
-
-The page PNGs are generated by `ingest_pdf_pages.py` and are safe to delete and
-re-generate from the source `release_2\{doc}\page_NNNN\page_NNNN.pdf` files.
-
-This matters for the Streamlit UI:
-
-- If the app runs on the same machine, you can pass the local path straight
-  into `st.video(path, start_time=...)` / `st.audio(...)` — that's the smoothest
-  playback experience and supports seeking.
-- If the app runs elsewhere, link out to the DVIDS page (`embedded_image_url`).
-  Direct CloudFront URLs work for download but seeking via HTTP from the
-  browser is hit-or-miss.
-- A third option: upload the local files to S3/R2/Vercel Blob and rewrite URLs.
-  Not done.
-
-If the file isn't found locally and the URL is the DVIDS page, **don't try to
-embed the CloudFront MP4 directly in `st.video()`** — DVIDS' `/download/asset/`
-endpoint is 403-gated, and the CloudFront URLs aren't stored in the DB. You'd
-need to re-scrape the page (see the `scrape_media_url` helper in
-`retry_release_2.py` if you want that pattern).
-
----
-
-## 10. Minimal working Streamlit app
-
-Drop this at `app.py` next to `embeddings_v2.py`, set the secrets, and run
-`streamlit run app.py`. It covers text query, source-type filter, threshold
-slider, and inline media playback with timestamp seeking.
-
-```python
-import os
-import re
-from pathlib import Path
-
-import psycopg
-import streamlit as st
-
-import embeddings_v2 as e
-
-USER_ID = "00000000-0000-0000-0000-000000000001"
-MEDIA_ROOT = Path(r"D:\divided\release_2\UAP_Release_2")  # change if elsewhere
-SOURCE_TYPES = ("video_chunk", "audio_clip", "pdf_page")
-
-st.set_page_config(page_title="UAP Archive Semantic Search", layout="wide")
-
-# --- bootstrap ---------------------------------------------------------------
-for k in ("DATABASE_URL", "GEMINI_API_KEY"):
-    if k in st.secrets:
-        os.environ.setdefault(k, st.secrets[k])
-    if not os.environ.get(k):
-        st.error(f"Missing {k} — add it to .streamlit/secrets.toml")
-        st.stop()
-
-@st.cache_resource
-def get_conn():
-    return psycopg.connect(os.environ["DATABASE_URL"])
-
-@st.cache_data(ttl=3600, show_spinner=False)
-def embed_query_text(text: str) -> list[float]:
-    # generate_text_embedding auto-wraps with format_query() and drops task_type.
-    return e.generate_text_embedding(text)
-
-@st.cache_data(ttl=3600, show_spinner=False)
-def embed_query_image(image_bytes: bytes, mime: str) -> list[float]:
-    import tempfile
-    suffix = "." + mime.split("/", 1)[1]
-    with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as f:
-        f.write(image_bytes)
-        path = f.name
-    try:
-        # image-only embed: same call for query and document side.
-        return e.generate_image_embedding(path)
-    finally:
-        os.unlink(path)
-
-def search(vec, *, source_type=None, release=None, limit=20, threshold=0.30):
-    # pgvector's psycopg adapter doesn't auto-cast list[float] to vector ---
-    # serialise to the textual '[a,b,c]' form and let Postgres cast.
-    vec_str = "[" + ",".join(f"{x:.6f}" for x in vec) + "]"
-    clauses = ["user_id = %s::uuid", "(embedding <=> %s::vector) <= %s"]
-    params  = [USER_ID, vec_str, 1 - threshold]
-    if source_type:
-        clauses.append("source_type = %s")
-        params.append(source_type)
-    if release:
-        clauses.append("release = %s")
-        params.append(release)
-    sql = f"""
-        SELECT source_type, source_id, parent_id, start_seconds, end_seconds,
-               embedded_image_url, embedded_text, release, release_date,
-               1 - (embedding <=> %s::vector) AS similarity
-        FROM embeddings
-        WHERE {' AND '.join(clauses)}
-        ORDER BY embedding <=> %s::vector
-        LIMIT %s
-    """
-    ordered = [vec_str, *params, vec_str, limit]
-    with get_conn().cursor() as cur:
-        cur.execute(sql, ordered)
-        cols = [d.name for d in cur.description]
-        return [dict(zip(cols, r)) for r in cur.fetchall()]
-
-_PAGE_RE = re.compile(r"^(.+):p(\d+)$")
-
-def local_media_path(row: dict) -> Path | None:
-    st_type = row["source_type"]
-    if st_type == "video_chunk":
-        asset_id = row["parent_id"].removeprefix("dvids_")
-        p = MEDIA_ROOT / "videos" / f"dvids_{asset_id}.mp4"
-        return p if p.exists() else None
-    if st_type == "audio_clip":
-        asset_id = row["parent_id"].removeprefix("dvids_")
-        for ext in ("m4a", "mp3", "mp4", "wav", "aac", "ogg"):
-            p = MEDIA_ROOT / "audio" / f"dvids_{asset_id}.{ext}"
-            if p.exists():
-                return p
-        return None
-    if st_type == "pdf_page":
-        m = _PAGE_RE.match(row["source_id"])
-        if not m:
-            return None
-        slug, page_num = m.group(1), int(m.group(2))
-        p = MEDIA_ROOT / "pages" / slug / f"page_{page_num:04d}.png"
-        return p if p.exists() else None
-    return None
-
-def page_number(row: dict) -> int | None:
-    if row["source_type"] != "pdf_page":
-        return None
-    m = _PAGE_RE.match(row["source_id"])
-    return int(m.group(2)) if m else None
-
-# --- UI ----------------------------------------------------------------------
-st.title("UAP Archive — Semantic Search")
-st.caption("Gemini 768-d embeddings, cosine similarity over Neon + pgvector.")
-
-with st.sidebar:
-    mode = st.radio("Query type", ["Text", "Image"], horizontal=True)
-    st_filter = st.selectbox("Source type", ["all", *SOURCE_TYPES])
-    release_filter = st.selectbox("Release", ["all", "PURSUE_2"])
-    threshold = st.slider("Min similarity", 0.0, 0.9, 0.30, 0.05)
-    limit = st.slider("Max results", 5, 50, 20)
-
-vec = None
-if mode == "Text":
-    q = st.text_input("Search query", placeholder="e.g. spherical UAP over water")
-    if q:
-        with st.spinner("Embedding query…"):
-            vec = embed_query_text(q)
-else:
-    up = st.file_uploader("Upload an image", type=["jpg", "jpeg", "png", "webp"])
-    if up:
-        st.image(up, width=240)
-        with st.spinner("Embedding image…"):
-            vec = embed_query_image(up.getvalue(), up.type)
-
-if vec is None:
-    st.info("Enter a query or upload an image.")
-    st.stop()
-
-with st.spinner("Searching Neon…"):
-    rows = search(
-        vec,
-        source_type=None if st_filter == "all" else st_filter,
-        release=None if release_filter == "all" else release_filter,
-        limit=limit,
-        threshold=threshold,
-    )
-
-if not rows:
-    st.warning("No matches above the similarity threshold. Try lowering it.")
-    st.stop()
-
-st.subheader(f"{len(rows)} result(s)")
-for r in rows:
-    with st.container(border=True):
-        c1, c2 = st.columns([4, 1])
-        with c1:
-            header = f"**[{r['parent_id']}]({r['embedded_image_url']})** · `{r['source_type']}` · sim **{r['similarity']:.3f}**"
-            page = page_number(r)
-            if page is not None:
-                header += f" · page {page}"
-            elif r["start_seconds"] is not None:
-                header += f" · {r['start_seconds']:.1f}s → {r['end_seconds']:.1f}s"
-            st.markdown(header)
-            if r["embedded_text"]:
-                st.write(r["embedded_text"][:600] + ("…" if len(r["embedded_text"]) > 600 else ""))
-            local = local_media_path(r)
-            if local and r["source_type"] == "video_chunk":
-                st.video(str(local), start_time=int(r["start_seconds"] or 0))
-            elif local and r["source_type"] == "audio_clip":
-                st.audio(str(local), start_time=int(r["start_seconds"] or 0))
-            elif local and r["source_type"] == "pdf_page":
-                st.image(str(local), use_container_width=True)
-                if r["embedded_image_url"]:
-                    st.link_button("Open full PDF on war.gov", r["embedded_image_url"])
-            elif r["embedded_image_url"]:
-                st.link_button("Open source", r["embedded_image_url"])
-        with c2:
-            st.metric("similarity", f"{r['similarity']:.3f}")
-            st.caption(f"{r['release']} · {r['release_date']}")
-```
-
----
-
-## 11. Gotchas / things that will trip you up
-
-- **Pooled vs direct Neon endpoint.** The user's connection string in the
-  earlier session was the `-pooler` host. For a long-lived Streamlit process
-  that reuses one connection across many queries, psycopg3 will eventually
-  promote a statement to a *named* prepared statement (default
-  `prepare_threshold=5`), which PgBouncer in transaction-pooling mode cannot
-  hold across transactions. Use the **direct** endpoint (host without
-  `-pooler`) or set `prepare_threshold=None` on the connection.
-- **Dimension must match.** The column is `VECTOR(768)`. Don't pass a 1536-dim
-  vector — it'll fail on the cast. If you ever switch to a different
-  `output_dimensionality`, you'll need to migrate the column.
-- **Instruction-in-prompt, not `task_type=`.** gemini-embedding-2 silently
-  ignores `EmbedContentConfig.task_type` on the consumer API and instead
-  expects the task to be expressed *inside the content*. Wrap documents as
-  `title: {title} | text: {body}` (via `e.format_document_text(...)`) and
-  queries as `task: search result | query: {q}` (via `e.format_query(...)`,
-  applied automatically by `e.generate_text_embedding`). Skipping this
-  produces noticeably worse ranking — the previous version of this corpus
-  ranked NASA audio narratives above DOW UAP video clips on the query
-  "instantaneous acceleration" because the asymmetric format wasn't applied;
-  the re-embed with proper wrapping put `dvids_1007707` at ranks 1–4.
-
-- **Vertex-only config options.** Three `EmbedContentConfig` fields exist in
-  the SDK but are rejected by the consumer Gemini API
-  (`"<option> parameter is not supported in Gemini API"`):
-  `document_ocr` (server-side PDF OCR), `audio_track_extraction` (pull audio
-  from video for the embed), and `auto_truncate`. They're only available via
-  Vertex AI. If you migrate to Vertex (`genai.Client(vertexai=True, project=...,
-  location=...)`), all three become usable and would let us simplify the
-  pipeline (no manual ffmpeg audio extraction, no manual OCR pre-step).
-- **`<=>` is distance, not similarity.** Lower = more similar. Always do
-  `1 - (embedding <=> query)` for a similarity score.
-- **HNSW recall.** The HNSW index is approximate. For exact ranking on small
-  result sets, you can `SET LOCAL hnsw.ef_search = 100;` before the query.
-- **First Neon query after idle is slow.** Neon auto-suspends idle databases;
-  expect ~500ms cold-start latency on the first request.
-- **Don't ship secrets.** `secrets.toml` should be `.gitignore`d. The keys from
-  the previous session are exposed in that chat transcript and should be
-  rotated.
-- **Streamlit `st.video` URL playback.** Local file paths work great and
-  support `start_time` seeking. Remote HTTP URLs are flaky for seeking —
-  prefer local files where possible.
-- **Audio for the NASA recordings.** The source assets on DVIDS are MP4
-  wrappers (large, ~200 MB each). The previous session extracted the audio
-  track to `.m4a` (a few MB each) and embedded *that*. Use the `.m4a` for
-  playback; ignore the source `.mp4` unless you want visual.
-- **`pgvector` + `psycopg3`: don't pass `list[float]` bare.** The pgvector
-  adapter doesn't auto-cast Python lists to the `vector` type. Either bind a
-  `numpy.ndarray`, or (what the example does) serialise the vector to the
-  textual form `'[a,b,c,…]'` and use `%s::vector` in the SQL. Forgetting this
-  fails with `operator does not exist: vector <=> double precision[]`.
-- **Text queries are biased toward text-rich modalities.** In this corpus,
-  any plain text query crowds the top with `audio_clip` and `pdf_page` rows
-  because their `embedded_text` is long (multi-sentence NASA narratives /
-  multi-line OCR), and because video chunks' multimodal vectors are pulled
-  toward visual neighborhoods that short text queries can't reach. Concrete
-  example: the query "instantaneous acceleration" returns 12 NASA Apollo /
-  Mercury audio rows in the top 12 — and **does not surface** the DVIDS clip
-  `dvids_1007707` whose title literally contains "instant acceleration". To
-  let video chunks compete: default to a `source_type` filter, present
-  **faceted results** (top-N per type side by side), or steer users toward
-  **image queries** (same-modality alignment with video frames).
-
----
-
-## 12. Quick test: does the database actually have what this doc claims?
-
-Run once before you start coding the UI:
-
-```python
-import os, psycopg
-with psycopg.connect(os.environ["DATABASE_URL"]) as c:
-    for row in c.execute(
-        "SELECT source_type, COUNT(*) AS rows, COUNT(DISTINCT parent_id) AS assets "
-        "FROM embeddings GROUP BY source_type ORDER BY source_type"
-    ).fetchall():
-        print(row)
-```
-
-Expected (as of the handoff): `('audio_clip', 27, 7)`, `('pdf_page', 126, 5)`, and `('video_chunk', 154, 49)` — total **307 rows** across **61 distinct parent_ids**, all `release='PURSUE_2'` / `release_date='2026-05-22'`.
-
----
-
-## 13. Suggested next steps for the Streamlit session
-
-1. Drop `embeddings_v2.py` and the `app.py` from §10 into a fresh folder.
-2. Create `.streamlit/secrets.toml` with `DATABASE_URL` and `GEMINI_API_KEY`.
-3. Run §12 to confirm DB connectivity.
-4. `streamlit run app.py` and test a few queries: `"spherical UAP over water"`,
-   `"high-speed maneuver"`, `"Apollo astronaut"`.
-5. Polish UI: result cards, thumbnails (DVIDS pages have poster images in
-   `og:image` if you want to scrape), pagination, multimodal query (already
-   stubbed in the example), per-result "show all chunks of this video" drilldown.
-6. Optional: add an admin tab that ingests new assets (re-uses `embeddings_v2`
-   plus the `retry_release_2.py` patterns).

analyzing.py

@@ -5,15 +5,14 @@ import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
 import seaborn as sns
-from uap_analyzer import UAPParser, UAPAnalyzer, UAPVisualizer, cramers_v
+from uap_analyzer import UAPParser, UAPAnalyzer, UAPVisualizer
 # import ChartGen
 # from ChartGen import ChartGPT
 from Levenshtein import distance
 from sklearn.model_selection import train_test_split
 from sklearn.metrics import confusion_matrix
-from scipy.stats import chi2_contingency
-import plotly.graph_objects as go
-from tqdm import tqdm
+from stqdm import stqdm
+stqdm.pandas()
 import streamlit.components.v1 as components
 from dateutil import parser
 from sentence_transformers import SentenceTransformer
@@ -23,12 +22,7 @@ import matplotlib.colors as mcolors
 import textwrap
 import datamapplot
 
-# Import enhanced utilities
-from utils.data_processing import DataProcessor
-from utils.visualization import UAP_Visualizer as Enhanced_Visualizer
-from utils.session_manager import SessionStateManager
-
-# st.set_option('deprecation.showPyplotGlobalUse', False)
+st.set_option('deprecation.showPyplotGlobalUse', False)
 
 from pandas.api.types import (
     is_categorical_dtype,
@@ -43,7 +37,35 @@ def load_data(file_path, key='df'):
     return pd.read_hdf(file_path, key=key)
 
 
-# Gemini Q&A moved to rag_search.py (the RAG search page) — see gemini_query there.
+def gemini_query(question, selected_data, gemini_key):
+
+    if question == "":
+        question = "Summarize the following data in relevant bullet points"
+
+    import pathlib
+    import textwrap
+
+    import google.generativeai as genai
+
+    from IPython.display import display
+    from IPython.display import Markdown
+
+
+    def to_markdown(text):
+        text = text.replace('•', '  *')
+        return Markdown(textwrap.indent(text, '> ', predicate=lambda _: True))
+    
+    # selected_data is a list
+    # remove empty
+
+    filtered = [str(x) for x in selected_data if str(x) != '' and x is not None]
+    # make a string
+    context = '\n'.join(filtered)
+
+    genai.configure(api_key=gemini_key)
+    query_model = genai.GenerativeModel('models/gemini-1.5-pro-latest')
+    response = query_model.generate_content([f"{question}\n Answer based on this context: {context}\n\n"])
+    return(response.text)
 
 def plot_treemap(df, column, top_n=32):
         # Get the value counts and the top N labels
@@ -100,69 +122,23 @@ def plot_treemap(df, column, top_n=32):
         ax.patch.set_alpha(0)
         return fig
 
-def plot_hist(df, column, bins=10, kde=True, figsize=(12, 6), color='orange', title=None):
-    """
-    Create a histogram with improved styling and error handling.
-    
-    Args:
-        df (pd.DataFrame): DataFrame containing the data
-        column (str): Column name to plot
-        bins (int): Number of bins for histogram
-        kde (bool): Whether to show kernel density estimation
-        figsize (tuple): Figure size
-        color (str): Color for the plot
-        title (str): Custom title for the plot
-    
-    Returns:
-        matplotlib.figure.Figure: The figure object
-    """
-    try:
-        fig, ax = plt.subplots(figsize=figsize, dpi=150)
-        
-        # Check if column exists and has data
-        if column not in df.columns:
-            ax.text(0.5, 0.5, f'Column "{column}" not found', ha='center', va='center', 
-                   transform=ax.transAxes, fontsize=12, color='red')
-            return fig
-            
-        data_series = df[column].dropna()
-        if len(data_series) == 0:
-            ax.text(0.5, 0.5, 'No data to plot', ha='center', va='center', 
-                   transform=ax.transAxes, fontsize=12, color='red')
-            return fig
-        
-        # Create histogram with improved bins calculation
-        if bins == 'auto':
-            bins = min(50, max(10, int(np.sqrt(len(data_series)))))
-        
-        sns.histplot(data=df, x=column, kde=kde, bins=bins, color=color, ax=ax, alpha=0.7)
-        
-        # Set title
-        ax.set_title(title or f'Distribution of {column}', color=color, fontweight='bold', fontsize=14)
-        ax.set_xlabel(column, color=color, fontsize=12)
-        ax.set_ylabel('Count', color=color, fontsize=12)
-        
-        # Style the plot
-        for spine in ax.spines.values():
-            spine.set_color(color)
-        
-        ax.tick_params(axis='both', colors=color)
-        ax.grid(True, alpha=0.3, color=color)
-        
+def plot_hist(df, column, bins=10, kde=True):
+        fig, ax = plt.subplots(figsize=(12, 6))
+        sns.histplot(data=df, x=column, kde=True, bins=bins,color='orange')
+        # set the ticks and frame in orange
+        ax.spines['bottom'].set_color('orange')
+        ax.spines['top'].set_color('orange')
+        ax.spines['right'].set_color('orange')
+        ax.spines['left'].set_color('orange')
+        ax.xaxis.label.set_color('orange')
+        ax.yaxis.label.set_color('orange')
+        ax.tick_params(axis='x', colors='orange')
+        ax.tick_params(axis='y', colors='orange')
+        ax.title.set_color('orange')
+
         # Set transparent background
         fig.patch.set_alpha(0)
         ax.patch.set_alpha(0)
-        
-        plt.tight_layout()
-        return fig
-        
-    except Exception as e:
-        fig, ax = plt.subplots(figsize=figsize)
-        ax.text(0.5, 0.5, f'Error creating histogram:\n{str(e)}', 
-               ha='center', va='center', transform=ax.transAxes, fontsize=12, color='red')
-        ax.set_title('Histogram Error', color='red')
-        fig.patch.set_alpha(0)
-        ax.patch.set_alpha(0)
         return fig
 
 
@@ -283,12 +259,6 @@ def plot_grouped_bar(df, x_columns, y_column, figsize=(12, 10), colors=None, tit
 
 
 def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame:
-    """
-    Enhanced filtering interface using the improved DataProcessor
-    """
-    return DataProcessor.filter_dataframe_enhanced(df, enable_quick_filters=False, enable_advanced_filters=True)
-
-def filter_dataframe_legacy(df: pd.DataFrame) -> pd.DataFrame:
     """
     Adds a UI on top of a dataframe to let viewers filter columns
 
@@ -380,104 +350,102 @@ def filter_dataframe_legacy(df: pd.DataFrame) -> pd.DataFrame:
                 st.pyplot(plot_hist(df_, column, bins=int(round(len(df_[column].unique())-1)/2)))
 
         elif is_object_dtype(df_[column]):
-            _orig_col = df_[column].copy()
-            _is_valid_date = False
             try:
-                df_[column] = pd.to_datetime(df_[column], errors='coerce')
+                df_[column] = pd.to_datetime(df_[column], infer_datetime_format=True, errors='coerce')
             except Exception:
                 try:
-                    df_[column] = df_[column].apply(lambda x: parser.parse(str(x)) if pd.notna(x) else pd.NaT)
+                    df_[column] = df_[column].apply(parser.parse)
                 except Exception:
                     pass
 
             if is_datetime64_any_dtype(df_[column]):
-                try:
-                    df_[column] = df_[column].dt.tz_localize(None)
-                except TypeError:
-                    df_[column] = df_[column].dt.tz_convert(None)
-                _valid = df_[column].dropna()
-                if not _valid.empty:
-                    min_date = _valid.min().date()
-                    max_date = _valid.max().date()
-                    if min_date != max_date:
-                        _is_valid_date = True
-                        user_date_input = right.date_input(
-                            f"Values for {column}",
-                            value=(min_date, max_date),
-                            min_value=min_date,
-                            max_value=max_date,
-                        )
-                        if len(user_date_input) == 2:
-                            user_date_input = tuple(map(pd.to_datetime, user_date_input))
-                            start_date, end_date = user_date_input
-
-                            time_units = {
-                                'year': df_[column].dt.year,
-                                'month': df_[column].dt.to_period('M'),
-                                'day': df_[column].dt.date
-                            }
-                            unique_counts = {unit: col.nunique() for unit, col in time_units.items()}
-                            closest_to_36 = min(unique_counts, key=lambda k: abs(unique_counts[k] - 36))
-
-                            grouped = df_.groupby(time_units[closest_to_36]).size().reset_index(name='count')
-                            grouped.columns = [column, 'count']
-
-                            if closest_to_36 == 'year':
-                                date_range = pd.date_range(start=f"{start_date.year}-01-01", end=f"{end_date.year}-12-31", freq='YS')
-                            elif closest_to_36 == 'month':
-                                date_range = pd.date_range(start=start_date.replace(day=1), end=end_date + pd.offsets.MonthEnd(0), freq='MS')
-                            else:
-                                date_range = pd.date_range(start=start_date, end=end_date, freq='D')
-
-                            complete_range = pd.DataFrame({column: date_range})
-                            if closest_to_36 == 'year':
-                                complete_range[column] = complete_range[column].dt.year
-                            elif closest_to_36 == 'month':
-                                complete_range[column] = complete_range[column].dt.to_period('M')
-
-                            final_data = pd.merge(complete_range, grouped, on=column, how='left').fillna(0)
-                            with st.status(f"Date Distributions: {column}", expanded=False) as stat:
-                                try:
-                                    st.pyplot(plot_bar(final_data, column, 'count'))
-                                except Exception as e:
-                                    st.error(f"Error plotting bar chart: {e}")
-                            df_ = df_.loc[df_[column].between(start_date, end_date)]
-
-                        date_column = column
-                        if date_column and filtered_columns:
-                            numeric_columns = [col for col in filtered_columns if is_numeric_dtype(df_[col])]
-                            categorical_columns = [col for col in filtered_columns if is_categorical_dtype(df_[col])]
-                            with st.status(f"Date Distribution: {column}", expanded=False) as stat:
-                                if numeric_columns:
-                                    try:
-                                        st.pyplot(plot_line(df_, date_column, numeric_columns))
-                                    except Exception as e:
-                                        st.error(f"Error plotting line chart: {e}")
-                                if categorical_columns:
-                                    try:
-                                        st.pyplot(plot_bar(df_, date_column, categorical_columns[0]))
-                                    except Exception as e:
-                                        st.error(f"Error plotting bar chart: {e}")
-
-            if not _is_valid_date:
-                df_[column] = _orig_col
-                _txt_col, _btn_col = right.columns([5, 1])
-                user_text_input = _txt_col.text_input(
-                    f"Substring or regex in {column}",
-                    key=f"regex_{column}",
+                df_[column] = df_[column].dt.tz_localize(None)
+                min_date = df_[column].min().date()
+                max_date = df_[column].max().date()
+                user_date_input = right.date_input(
+                    f"Values for {column}",
+                    value=(min_date, max_date),
+                    min_value=min_date,
+                    max_value=max_date,
                 )
-                _drop_null = _btn_col.checkbox("Drop nulls", key=f"dropnull_{column}")
-                if user_text_input:
-                    try:
-                        mask = df_[column].astype(str).str.contains(user_text_input, case=False, na=False, regex=True)
-                    except Exception:
-                        mask = df_[column].astype(str).str.contains(user_text_input, case=False, na=False, regex=False)
-                    df_ = df_.loc[mask]
-                    filtered_columns.append(column)
-                if _drop_null:
-                    df_ = df_.loc[df_[column].notna() & (df_[column].astype(str).str.strip() != '')]
-                    if column not in filtered_columns:
-                        filtered_columns.append(column)
+                # if len(user_date_input) == 2:
+                #     start_date, end_date = user_date_input
+                #     df_ = df_.loc[df_[column].dt.date.between(start_date, end_date)]
+                if len(user_date_input) == 2:
+                    user_date_input = tuple(map(pd.to_datetime, user_date_input))
+                    start_date, end_date = user_date_input
+
+                    # Determine the most appropriate time unit for plot
+                    time_units = {
+                        'year': df_[column].dt.year,
+                        'month': df_[column].dt.to_period('M'),
+                        'day': df_[column].dt.date
+                    }
+                    unique_counts = {unit: col.nunique() for unit, col in time_units.items()}
+                    closest_to_36 = min(unique_counts, key=lambda k: abs(unique_counts[k] - 36))
+
+                    # Group by the most appropriate time unit and count occurrences
+                    grouped = df_.groupby(time_units[closest_to_36]).size().reset_index(name='count')
+                    grouped.columns = [column, 'count']
+
+                    # Create a complete date range
+                    if closest_to_36 == 'year':
+                        date_range = pd.date_range(start=f"{start_date.year}-01-01", end=f"{end_date.year}-12-31", freq='YS')
+                    elif closest_to_36 == 'month':
+                        date_range = pd.date_range(start=start_date.replace(day=1), end=end_date + pd.offsets.MonthEnd(0), freq='MS')
+                    else:  # day
+                        date_range = pd.date_range(start=start_date, end=end_date, freq='D')
+
+                    # Create a DataFrame with the complete date range
+                    complete_range = pd.DataFrame({column: date_range})
+
+                    # Convert the date column to the appropriate format based on closest_to_36
+                    if closest_to_36 == 'year':
+                        complete_range[column] = complete_range[column].dt.year
+                    elif closest_to_36 == 'month':
+                        complete_range[column] = complete_range[column].dt.to_period('M')
+
+                    # Merge the complete range with the grouped data
+                    final_data = pd.merge(complete_range, grouped, on=column, how='left').fillna(0)
+
+                    with st.status(f"Date Distributions: {column}", expanded=False) as stat:
+                        try:
+                            st.pyplot(plot_bar(final_data, column, 'count'))
+                        except Exception as e:
+                            st.error(f"Error plotting bar chart: {e}")
+                    
+                    df_ = df_.loc[df_[column].between(start_date, end_date)]
+
+                date_column = column
+
+                if date_column and filtered_columns:
+                    numeric_columns = [col for col in filtered_columns if is_numeric_dtype(df_[col])]
+                    if numeric_columns:
+                        fig = plot_line(df_, date_column, numeric_columns)
+                        #st.pyplot(fig)
+                    # now to deal with categorical columns
+                    categorical_columns = [col for col in filtered_columns if is_categorical_dtype(df_[col])]
+                    if categorical_columns:
+                        fig2 = plot_bar(df_, date_column, categorical_columns[0])
+                        #st.pyplot(fig2)
+                    with st.status(f"Date Distribution: {column}", expanded=False) as stat:
+                        try:
+                            st.pyplot(fig)
+                        except Exception as e:
+                            st.error(f"Error plotting line chart: {e}")
+                            pass
+                        try:
+                            st.pyplot(fig2)
+                        except Exception as e:
+                            st.error(f"Error plotting bar chart: {e}")
+
+
+        else:
+            user_text_input = right.text_input(
+                f"Substring or regex in {column}",
+            )
+            if user_text_input:
+                df_ = df_[df_[column].astype(str).str.contains(user_text_input)]
     # write len of df after filtering with % of original
     st.write(f"{len(df_)} rows ({len(df_) / len(df) * 100:.2f}%)")
     return df_
@@ -506,658 +474,6 @@ def merge_clusters(df, column):
     df.__dict__['string_labels'] = updated_string_labels
     return updated_string_labels
 
-# ── Categorical Association Explorer (Cramér's V) ───────────────────────────
-# Bands columns by unique-value count so genuinely categorical columns are
-# auto-selected for a fast first-pass Cramér's V, while high-cardinality
-# columns (free-text, IDs, numerics) are excluded by default.
-
-def _safe_nunique(series):
-    """nunique that tolerates unhashable cells (lists/dicts) by stringifying."""
-    try:
-        return int(series.nunique(dropna=True))
-    except TypeError:
-        return int(series.astype(str).nunique(dropna=True))
-
-
-def band_columns(df, high_threshold=30):
-    """Bucket columns into categorical bands by cardinality.
-
-    Returns (bands, nunique_map):
-      bands = {"binary": [...], "low": [...], "medium": [...], "high": [...],
-               "constant": [...]}
-        - binary    : exactly 2 distinct values
-        - low       : 3 .. 9
-        - medium    : 10 .. high_threshold-1
-        - high       : >= high_threshold (treated as non-categorical, excluded)
-        - constant  : <= 1 distinct value (useless for association)
-    """
-    bands = {"binary": [], "low": [], "medium": [], "high": [], "constant": []}
-    nunique_map = {}
-    for c in df.columns:
-        nu = _safe_nunique(df[c])
-        nunique_map[c] = nu
-        if nu <= 1:
-            bands["constant"].append(c)
-        elif nu == 2:
-            bands["binary"].append(c)
-        elif nu <= 9:
-            bands["low"].append(c)
-        elif nu < high_threshold:
-            bands["medium"].append(c)
-        else:
-            bands["high"].append(c)
-    return bands, nunique_map
-
-
-# Single canonical label for every flavour of "absent" so missingness isn't
-# fragmented into nan / None / <NA> / NaT / "" as separate categories.
-_MISSING_LABEL = "(missing)"
-_NULL_STR_TOKENS = {"nan", "none", "null", "<na>", "nat", ""}
-
-
-def _coalesce(series):
-    """Stringify a column and fold all null representations into one
-    `(missing)` category (fixes fragmentation). Never mutates the source."""
-    s = series.astype(str).str.strip()
-    return s.mask(s.str.lower().isin(_NULL_STR_TOKENS), _MISSING_LABEL)
-
-
-def _pair_series(df, c1, c2, drop_missing):
-    """Coalesced (and optionally complete-case) aligned pair of columns."""
-    a, b = _coalesce(df[c1]), _coalesce(df[c2])
-    if drop_missing:
-        keep = (a != _MISSING_LABEL) & (b != _MISSING_LABEL)
-        a, b = a[keep], b[keep]
-    return a, b
-
-
-def compute_cramers_v_df(df, cols, drop_missing=False):
-    """Symmetric Cramér's V matrix over `cols` (diagonal = 1.0). Each cell is
-    computed once and mirrored. Nulls are coalesced to a single `(missing)`
-    category; when `drop_missing` is set, each pair is reduced to its
-    complete cases (note: per-cell N then varies across the matrix)."""
-    cv = pd.DataFrame(index=cols, columns=cols, data=np.nan, dtype=float)
-    cache = {c: _coalesce(df[c]) for c in cols}
-    for i, c1 in enumerate(cols):
-        cv.at[c1, c1] = 1.0
-        for c2 in cols[i + 1:]:
-            a, b = cache[c1], cache[c2]
-            if drop_missing:
-                keep = (a != _MISSING_LABEL) & (b != _MISSING_LABEL)
-                a, b = a[keep], b[keep]
-            v = 0.0 if len(a) == 0 else cramers_v(pd.crosstab(a, b))
-            cv.at[c1, c2] = v
-            cv.at[c2, c1] = v
-    return cv
-
-
-_CV_TOL = 1e-6  # treat values within this of 0 / 1 as trivial
-
-
-def _is_trivial_v(v, tol=_CV_TOL):
-    """True for a Cramér's V that's effectively 0 (no association — likely
-    null/constant) or 1 (perfect association — likely a duplicate column)."""
-    return (v <= tol) or (v >= 1.0 - tol)
-
-
-def high_correlation_columns(cv_df, strong_threshold=0.30, exclude_trivial=True):
-    """Columns reaching Cramér's V ≥ strong_threshold with ≥1 other column.
-
-    Mirrors the pass-2 pre-fill logic in render_cramers_v_explorer so the
-    analysis form can reuse whatever the explorer last computed. Trivial pairs
-    (V≈0 null/constant, V≈1 duplicate) are skipped when exclude_trivial is set.
-    Returns [] if cv_df is None or empty.
-    """
-    if cv_df is None or getattr(cv_df, "empty", True):
-        return []
-    out = []
-    for col in cv_df.columns:
-        others = cv_df[col].drop(labels=[col], errors="ignore")
-        for v in others:
-            if pd.isna(v):
-                continue
-            v = float(v)
-            if exclude_trivial and _is_trivial_v(v):
-                continue
-            if v >= strong_threshold:
-                out.append(col)
-                break
-    return out
-
-
-def _cramers_pairs_table(cv_df, exclude_trivial=True):
-    """Ranked long-form table of unique off-diagonal pairs, strongest first.
-
-    When `exclude_trivial` is set, pairs with Cramér's V ≈ 0 (no association,
-    likely null/constant) or ≈ 1 (perfect association, likely duplicate
-    columns) are dropped. Returns (table, n_excluded)."""
-    rows, n_excluded = [], 0
-    cols = list(cv_df.columns)
-    for i, c1 in enumerate(cols):
-        for c2 in cols[i + 1:]:
-            v = cv_df.at[c1, c2]
-            if pd.isna(v):
-                continue
-            v = float(v)
-            if exclude_trivial and _is_trivial_v(v):
-                n_excluded += 1
-                continue
-            rows.append({"Variable A": c1, "Variable B": c2,
-                         "Cramér's V": round(v, 3)})
-    out = pd.DataFrame(rows)
-    if not out.empty:
-        out = out.sort_values("Cramér's V", ascending=False).reset_index(drop=True)
-    return out, n_excluded
-
-
-def _interactive_cv_heatmap(cv_df, title, key):
-    """Clickable lower-triangle Cramér's V heatmap. Returns the (row, col)
-    column names of the most recently clicked cell, or None.
-
-    Clicking a cell drives the contingency drill-down below the chart.
-    """
-    cols = list(cv_df.columns)
-    n = len(cols)
-    z = cv_df.astype(float).values.copy()
-    # Mask the strict upper triangle so each pair shows once (lower triangle).
-    z[np.triu(np.ones_like(z, dtype=bool), k=1)] = np.nan
-    annotate = n <= 25
-    fig = go.Figure(go.Heatmap(
-        z=z, x=cols, y=cols, colorscale="RdBu_r", zmin=0, zmax=1,
-        colorbar=dict(title="Cramér's V"),
-        hovertemplate="A (row): %{y}<br>B (col): %{x}<br>V: %{z:.3f}"
-                      "<extra></extra>",
-        texttemplate="%{z:.2f}" if annotate else None,
-        textfont={"size": 8},
-    ))
-    fig.update_layout(
-        title=f"{title} — click a cell (or pick a pair below) to inspect categories",
-        height=max(450, 24 * n + 160),
-        yaxis=dict(autorange="reversed", scaleanchor="x", constrain="domain"),
-        template="plotly_dark",
-        paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)",
-        margin={"l": 10, "r": 10, "t": 60, "b": 10},
-    )
-    try:
-        event = st.plotly_chart(fig, use_container_width=True,
-                                on_select="rerun", key=key)
-    except TypeError:
-        # Older Streamlit without on_select — render non-interactive.
-        st.plotly_chart(fig, use_container_width=True, key=key)
-        return None
-    # Selection payload shape varies across Streamlit versions; dig defensively.
-    sel = {}
-    if event is not None:
-        sel = (event.get("selection") if hasattr(event, "get") else None) or {}
-    pts = sel.get("points", []) if hasattr(sel, "get") else []
-    if pts:
-        p = pts[-1]
-        c_row, c_col = p.get("y"), p.get("x")
-        if c_row in cv_df.columns and c_col in cv_df.columns and c_row != c_col:
-            return c_row, c_col
-    return None
-
-
-def _pair_drilldown(df, cv_df, pairs_df, key_prefix, clicked, drop_missing=False):
-    """Render Variable-A / Variable-B selectors (defaulting to the strongest
-    pair, or to a clicked heatmap cell) and the contingency drill-down. Works
-    even when heatmap click events don't fire — the selectors are the reliable
-    path; a click just pre-sets them."""
-    cols = list(cv_df.columns)
-    if len(cols) < 2:
-        return
-    ka, kb = f"{key_prefix}_a", f"{key_prefix}_b"
-
-    # Strongest pair from the (already filtered) ranked table → sensible default.
-    if pairs_df is not None and not pairs_df.empty:
-        default_a = pairs_df.iloc[0]["Variable A"]
-        default_b = pairs_df.iloc[0]["Variable B"]
-    else:
-        default_a, default_b = cols[0], cols[1]
-
-    # Seed defaults once; reset if a stale value isn't in the current matrix.
-    if ka not in st.session_state or st.session_state[ka] not in cols:
-        st.session_state[ka] = default_a
-    if kb not in st.session_state or st.session_state[kb] not in cols:
-        st.session_state[kb] = default_b
-    # A heatmap click overrides the current selection.
-    if clicked:
-        st.session_state[ka], st.session_state[kb] = clicked[0], clicked[1]
-
-    cA, cB = st.columns(2)
-    with cA:
-        a = st.selectbox("Variable A (row)", cols, key=ka)
-    with cB:
-        b = st.selectbox("Variable B (col)", cols, key=kb)
-    if a == b:
-        st.info("Pick two different columns to see their co-occurring categories.")
-        return
-    _render_contingency(df, a, b, float(cv_df.at[a, b]), key_prefix, drop_missing)
-
-
-def _render_contingency(df, c1, c2, v, key_prefix, drop_missing=False):
-    """Show which category VALUES co-vary for the chosen column pair: a
-    crosstab the user can view as counts, row/column %, or standardized
-    residuals (the cells that drive the association). Nulls are coalesced to a
-    single `(missing)` category, or dropped (complete cases) when requested."""
-    st.markdown(f"#### 🔬 `{c1}`  ✕  `{c2}` — Cramér's V = **{v:.3f}**")
-    a, b = _pair_series(df, c1, c2, drop_missing)
-    n_used = len(a)
-    n_total = len(df)
-    if drop_missing:
-        st.caption(f"Complete cases: **{n_used:,}** of {n_total:,} rows "
-                   f"({n_used / n_total * 100:.1f}%) — rows missing either "
-                   "column dropped.")
-    else:
-        st.caption(f"All **{n_total:,}** rows; nulls folded into one "
-                   "`(missing)` category.")
-    if n_used == 0:
-        st.info("No rows left for this pair after dropping missing values.")
-        return
-    ct = pd.crosstab(a, b)
-    if ct.size == 0:
-        st.info("Empty crosstab for this pair.")
-        return
-
-    # Cap very large crosstabs to the most frequent values per axis.
-    MAXC = 30
-    note = ""
-    r, k = ct.shape
-    if r > MAXC or k > MAXC:
-        top_r = a.value_counts().nlargest(MAXC).index
-        top_k = b.value_counts().nlargest(MAXC).index
-        ct = ct.loc[ct.index.isin(top_r), ct.columns.isin(top_k)]
-        note = f" · showing top {MAXC} values per axis"
-
-    # χ²-expected-count health check (the residual approximation wants ≥5).
-    if min(ct.shape) >= 2:
-        try:
-            _, _, _, _exp = chi2_contingency(ct)
-            _low = float((_exp < 5).mean()) * 100
-            if _low > 20:
-                st.warning(
-                    f"⚠️ {_low:.0f}% of cells have an expected count < 5 — the "
-                    "χ² / standardized-residual approximation is unstable here. "
-                    "Treat residuals as indicative, not exact."
-                )
-        except ValueError:
-            pass
-
-    view = st.radio(
-        "Cell values", ["Counts", "Row %", "Column %", "Std. residuals"],
-        horizontal=True, key=f"{key_prefix}_ctview",
-        help="Std. residual = (observed − expected) / √expected. |residual| > 2 "
-             "marks a value-pair that co-occurs far more (blue) or less (red) "
-             "than chance — these are what make the two columns covariable.",
-    )
-
-    if view == "Counts":
-        mat, fmt, cs, zmid = ct, ".0f", "Blues", None
-    elif view == "Row %":
-        mat = ct.div(ct.sum(axis=1).replace(0, np.nan), axis=0) * 100
-        fmt, cs, zmid = ".0f", "Blues", None
-    elif view == "Column %":
-        mat = ct.div(ct.sum(axis=0).replace(0, np.nan), axis=1) * 100
-        fmt, cs, zmid = ".0f", "Blues", None
-    else:  # Std. residuals
-        chi2, p, dof, expected = chi2_contingency(ct)
-        with np.errstate(divide="ignore", invalid="ignore"):
-            resid = (ct.values - expected) / np.sqrt(expected)
-        mat = pd.DataFrame(np.nan_to_num(resid), index=ct.index, columns=ct.columns)
-        fmt, cs, zmid = ".1f", "RdBu", 0
-
-    rows, kcols = mat.shape
-    show_text = rows <= 25 and kcols <= 25
-    fig = go.Figure(go.Heatmap(
-        z=mat.values,
-        x=[str(c) for c in mat.columns], y=[str(i) for i in mat.index],
-        colorscale=cs, zmid=zmid,
-        texttemplate=f"%{{z:{fmt}}}" if show_text else None,
-        textfont={"size": 8},
-        hovertemplate=f"{c1}=%{{y}}<br>{c2}=%{{x}}<br>%{{z:{fmt}}}<extra></extra>",
-    ))
-    fig.update_layout(
-        title=f"{c1} ✕ {c2} — {view}{note}",
-        height=max(380, 22 * rows + 160),
-        xaxis_title=c2, yaxis_title=c1,
-        yaxis=dict(autorange="reversed"),
-        template="plotly_dark",
-        paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)",
-    )
-    st.plotly_chart(fig, use_container_width=True, key=f"{key_prefix}_ctfig")
-
-    if view == "Std. residuals":
-        # Surface the strongest covariable value-pairs as a ranked table.
-        rl = [
-            {f"{c1}": i, f"{c2}": j, "std_residual": round(float(mat.at[i, j]), 2),
-             "count": int(ct.at[i, j])}
-            for i in mat.index for j in mat.columns
-            if abs(mat.at[i, j]) >= 2
-        ]
-        if rl:
-            rl_df = pd.DataFrame(rl)
-            rl_df = (rl_df.assign(_abs=rl_df["std_residual"].abs())
-                          .sort_values("_abs", ascending=False)
-                          .drop(columns="_abs")
-                          .reset_index(drop=True))
-            st.caption("Value-pairs that co-occur more/less than chance "
-                       "(|std. residual| ≥ 2):")
-            st.dataframe(rl_df.head(30), hide_index=True, use_container_width=True)
-        else:
-            st.caption("No value-pair exceeds |std. residual| ≥ 2 — the "
-                       "association is spread evenly rather than driven by "
-                       "specific value combinations.")
-
-
-def render_cramers_v_explorer(df):
-    """Cardinality-banded, two-pass Cramér's V association explorer.
-
-    Pass 1 auto-runs over every column the cardinality heuristic flags as
-    categorical (binary + low + medium bands). Pass 2 lets the user refine the
-    set — pre-filled with whichever columns carried a strong association.
-    """
-    st.markdown("### 🎲 Categorical Association Explorer (Cramér's V)")
-    st.caption(
-        "Columns are banded by their number of distinct values. Genuinely "
-        "categorical columns are auto-selected for a first-pass association "
-        "heatmap; high-cardinality columns (free-text, IDs, numerics) are "
-        "treated as non-categorical and excluded by default."
-    )
-
-    cc1, cc2 = st.columns(2)
-    with cc1:
-        high_threshold = int(st.number_input(
-            "Non-categorical threshold (distinct values)",
-            min_value=3, max_value=1000, value=30, step=1,
-            key="cv_high_threshold",
-            help="Columns with this many or more distinct values are treated "
-                 "as non-categorical and land in the (opt-in) High band.",
-        ))
-    with cc2:
-        strong_threshold = float(st.slider(
-            "Strong-association threshold (for pass 2 pre-fill)",
-            0.0, 1.0, 0.30, 0.05, key="cv_strong_threshold",
-            help="In pass 2, columns are pre-selected if they reach at least "
-                 "this Cramér's V with any other column in pass 1.",
-        ))
-
-    fc1, fc2 = st.columns([2, 3])
-    with fc1:
-        exclude_trivial = st.checkbox(
-            "Filter out V≈0 and V≈1 pairs (likely null / duplicate)",
-            value=True, key="cv_exclude_trivial",
-            help="Drops pairs with no association (V≈0 — often constant/null "
-                 "columns) and perfect association (V≈1 — often duplicate "
-                 "columns) from the ranked tables and the pass-2 pre-fill.",
-        )
-    with fc2:
-        missing_mode = st.radio(
-            "Missing values",
-            ["Treat as “(missing)” category", "Drop missing (complete cases)"],
-            horizontal=True, key="cv_missing_mode",
-            help="Coalesce: all nulls (nan/None/<NA>) become one “(missing)” "
-                 "level — keeps every row, lets you study co-missingness. "
-                 "Drop: each pair uses only rows where both are present "
-                 "(unbiased only if data is missing-completely-at-random).",
-        )
-    drop_missing = missing_mode.startswith("Drop")
-
-    bands, nunique_map = band_columns(df, high_threshold=high_threshold)
-
-    band_meta = [
-        ("binary", "Binary (2 values)", True),
-        ("low", "Low (3–9 values)", True),
-        ("medium", f"Medium (10–{high_threshold - 1} values)", True),
-        ("high", f"High (≥{high_threshold} — non-categorical)", False),
-    ]
-
-    selected = []
-    for key, label, prefill in band_meta:
-        opts = bands[key]
-        if not opts:
-            continue
-        # Sort options by cardinality so the most-categorical appear first.
-        opts = sorted(opts, key=lambda c: nunique_map[c])
-        default = opts if prefill else []
-        picked = st.multiselect(
-            f"{label} — {len(opts)} column(s)",
-            options=opts,
-            default=default,
-            key=f"cv_band_{key}",
-            help="Pre-filled and included in pass 1." if prefill
-                 else "Not pre-filled — tick to opt in (crosstabs can be large).",
-        )
-        selected.extend(picked)
-
-    if bands["constant"]:
-        st.caption(
-            f":grey[Skipped {len(bands['constant'])} constant/empty column(s): "
-            f"{', '.join(bands['constant'][:8])}"
-            f"{'…' if len(bands['constant']) > 8 else ''}]"
-        )
-
-    # De-dup while preserving order.
-    selected = list(dict.fromkeys(selected))
-    n_sel = len(selected)
-    n_pairs = n_sel * (n_sel - 1) // 2
-    st.caption(f"**{n_sel}** column(s) selected → **{n_pairs:,}** pairwise "
-               "Cramér's V computations.")
-    if n_sel > 40:
-        st.warning(
-            f"{n_sel} columns selected — pass 1 computes {n_pairs:,} crosstabs "
-            "and may be slow. Consider trimming the Medium band."
-        )
-
-    if st.button("▶️ Run Cramér's V — Pass 1", type="primary",
-                 disabled=n_sel < 2, key="cv_run_pass1"):
-        with st.spinner(f"Computing Cramér's V over {n_sel} columns…"):
-            cv_df = compute_cramers_v_df(df, selected, drop_missing=drop_missing)
-        st.session_state["cv_pass1_df"] = cv_df
-        st.session_state.pop("cv_pass2_df", None)
-
-    cv1 = st.session_state.get("cv_pass1_df")
-    if cv1 is not None:
-        st.markdown("#### Pass 1 — auto-selected categoricals")
-        clicked1 = _interactive_cv_heatmap(cv1, "Cramér's V — Pass 1", "cv_hm1")
-        pairs1, n_excl1 = _cramers_pairs_table(cv1, exclude_trivial=exclude_trivial)
-        # Keep only the download button in the main flow; tuck the pair
-        # comparison drill-down and the ranked table behind a dropdown.
-        if not pairs1.empty:
-            st.download_button(
-                "⬇️ Download pass-1 pairs (CSV)", pairs1.to_csv(index=False),
-                "cramers_v_pass1.csv", "text/csv", key="cv_dl_pass1",
-            )
-        # Popover, not an expander: this whole explorer is rendered inside the
-        # "Categorical Association Explorer" expander, and Streamlit forbids
-        # nesting an expander inside another expander.
-        with st.popover("🔎 Pair comparison & ranked table", use_container_width=True):
-            _pair_drilldown(df, cv1, pairs1, "cv_dd1", clicked1, drop_missing)
-            if n_excl1:
-                st.caption(f":grey[Filtered {n_excl1} trivial pair(s) "
-                           "(V≈0 null/constant or V≈1 duplicate).]")
-            if not pairs1.empty:
-                st.caption("Strongest associations (pass 1):")
-                st.dataframe(pairs1.head(25), hide_index=True, use_container_width=True)
-            else:
-                st.info("No non-trivial associations found in pass 1.")
-
-        # ── Pass 2 — refine ────────────────────────────────────────────────
-        st.divider()
-        st.markdown("#### Pass 2 — refine")
-        # A column qualifies if it reaches the strong threshold with another
-        # column — but trivial values (≈1 duplicates, ≈0) are skipped when the
-        # filter is on, so duplicate-driven columns don't auto-fill pass 2.
-        def _qualifies(col):
-            others = cv1[col].drop(labels=[col])
-            for v in others:
-                if pd.isna(v):
-                    continue
-                v = float(v)
-                if exclude_trivial and _is_trivial_v(v):
-                    continue
-                if v >= strong_threshold:
-                    return True
-            return False
-        strong_cols = [c for c in cv1.columns if _qualifies(c)]
-        st.caption(
-            f"Pre-filled with the **{len(strong_cols)}** column(s) reaching "
-            f"Cramér's V ≥ {strong_threshold:.2f} with any other column in "
-            "pass 1. Edit the set and re-run for a focused heatmap."
-        )
-        refine_sel = st.multiselect(
-            "Columns for pass 2",
-            options=list(cv1.columns),
-            default=strong_cols or list(cv1.columns),
-            key="cv_refine_sel",
-        )
-        refine_sel = list(dict.fromkeys(refine_sel))
-        if st.button("🔬 Run Cramér's V — Pass 2 (refined)",
-                     disabled=len(refine_sel) < 2, key="cv_run_pass2"):
-            with st.spinner(f"Re-computing over {len(refine_sel)} columns…"):
-                # Sub-select from the pass-1 matrix when possible; recompute
-                # only if the user somehow added columns not in pass 1.
-                if set(refine_sel).issubset(set(cv1.columns)):
-                    cv2 = cv1.loc[refine_sel, refine_sel]
-                else:
-                    cv2 = compute_cramers_v_df(df, refine_sel, drop_missing=drop_missing)
-            st.session_state["cv_pass2_df"] = cv2
-
-        cv2 = st.session_state.get("cv_pass2_df")
-        if cv2 is not None:
-            clicked2 = _interactive_cv_heatmap(cv2, "Cramér's V — Pass 2 (refined)", "cv_hm2")
-            pairs2, n_excl2 = _cramers_pairs_table(cv2, exclude_trivial=exclude_trivial)
-            # Same as pass 1: only the download button stays in the main flow.
-            if not pairs2.empty:
-                st.download_button(
-                    "⬇️ Download pass-2 pairs (CSV)", pairs2.to_csv(index=False),
-                    "cramers_v_pass2.csv", "text/csv", key="cv_dl_pass2",
-                )
-            with st.popover("🔎 Pair comparison & ranked table (refined)", use_container_width=True):
-                _pair_drilldown(df, cv2, pairs2, "cv_dd2", clicked2, drop_missing)
-                if n_excl2:
-                    st.caption(f":grey[Filtered {n_excl2} trivial pair(s) "
-                               "(V≈0 null/constant or V≈1 duplicate).]")
-                if not pairs2.empty:
-                    st.dataframe(pairs2.head(40), hide_index=True,
-                                 use_container_width=True)
-                else:
-                    st.info("No non-trivial associations in the refined set.")
-
-
-def render_categorical_flow_sankey(df):
-    """All-with-all categorical flow Sankey for the Statistical Analysis section.
-
-    Unlike the cross-DB matcher in rag_search.py (which pairs records by
-    similarity), this links every value of each chosen column to every
-    co-occurring value of the next column, with link width = the number of rows
-    sharing that combination (a chained crosstab). No pairwise matching — just
-    all-with-all co-occurrence counts.
-    """
-    st.markdown("### 🌊 Categorical Flow (Sankey)")
-    st.caption(
-        "Pick two or more categorical columns as ordered levels (left → right). "
-        "Every value is linked to every co-occurring value of the next level, "
-        "with link width = the number of rows sharing that combination — no "
-        "pairwise matching, just all-with-all co-occurrence counts."
-    )
-
-    n = len(df)
-    if n == 0:
-        st.info("No rows to chart.")
-        return
-
-    def _safe_nunique(s):
-        # Columns of dicts/lists are unhashable; fall back to string form.
-        try:
-            return s.nunique(dropna=False)
-        except TypeError:
-            return s.astype(str).nunique(dropna=False)
-
-    cat_like = [
-        c for c in df.columns
-        if 1 < _safe_nunique(df[c]) <= max(50, int(n * 0.5))
-    ]
-    if len(cat_like) < 2:
-        st.info("Need at least two categorical-like columns for a flow diagram.")
-        return
-
-    sc1, sc2 = st.columns([3, 1])
-    with sc1:
-        cols = st.multiselect(
-            "Levels (left → right, in order)",
-            options=list(df.columns),
-            default=cat_like[:3],
-            key="sankey_flow_cols",
-        )
-    with sc2:
-        top_n = int(st.number_input(
-            "Top-N / level", min_value=2, max_value=50, value=12, step=1,
-            key="sankey_flow_topn",
-            help="Keep only the most frequent N values per level; the rest are "
-                 "lumped into “Other” so the diagram stays readable.",
-        ))
-
-    if len(cols) < 2:
-        st.info("Select at least two columns.")
-        return
-
-    # Normalise: string-cast, fill missing, cap to top-N per level (+ “Other”).
-    work = pd.DataFrame(index=df.index)
-    for c in cols:
-        s = df[c].astype(str).where(df[c].notna(), "(missing)")
-        keep = s.value_counts().head(top_n).index
-        work[c] = s.where(s.isin(keep), "Other")
-
-    # Namespace nodes by (level, value) so identical values in different levels
-    # never merge into one node.
-    PALETTE = ["#f97316", "#22c55e", "#3b82f6", "#a855f7", "#ec4899",
-               "#14b8a6", "#eab308", "#ef4444"]
-    index, node_label, node_color = {}, [], []
-    L = len(cols)
-    max_per_level = 1
-    for level, c in enumerate(cols):
-        vc = work[c].value_counts()
-        max_per_level = max(max_per_level, len(vc))
-        for v in vc.index:
-            key = (level, v)
-            if key not in index:
-                index[key] = len(node_label)
-                node_label.append(str(v))
-                node_color.append(PALETTE[level % len(PALETTE)])
-
-    # Links: co-occurrence counts between consecutive levels (the "all with all").
-    srcs, tgts, vals = [], [], []
-    for level in range(L - 1):
-        a, b = cols[level], cols[level + 1]
-        counts = work.groupby([a, b]).size().reset_index(name="value")
-        for _, row in counts.iterrows():
-            srcs.append(index[(level, row[a])])
-            tgts.append(index[(level + 1, row[b])])
-            vals.append(int(row["value"]))
-
-    def _rgba(hex_color, alpha=0.35):
-        h = hex_color.lstrip("#")
-        return f"rgba({int(h[0:2], 16)},{int(h[2:4], 16)},{int(h[4:6], 16)},{alpha})"
-
-    fig = go.Figure(go.Sankey(
-        arrangement="snap",
-        node=dict(label=node_label, color=node_color, pad=14, thickness=18),
-        link=dict(source=srcs, target=tgts, value=vals,
-                  color=[_rgba(node_color[s]) for s in srcs]),
-    ))
-    fig.update_layout(
-        template="plotly_dark",
-        paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)",
-        font_size=12, height=max(420, 60 + 24 * max_per_level),
-        margin=dict(l=10, r=10, t=20, b=10),
-    )
-    st.plotly_chart(fig, use_container_width=True)
-    st.caption(
-        f"{' → '.join(cols)} · {len(node_label)} nodes · {len(srcs)} links "
-        "(all-with-all co-occurrence, no matching)."
-    )
-
-
 def analyze_and_predict(data, analyzers, col_names, clusters):
     visualizer = UAPVisualizer()
     new_data = pd.DataFrame()
@@ -1231,6 +547,9 @@ data_path = 'parsed_files_distance_embeds.h5'
 
 my_dataset = st.file_uploader("Upload Parsed DataFrame", type=["csv", "xlsx"])
 if my_dataset is not None:
+    
+    if parsed: # save space by cleaning default dataset
+        parsed = None
     try:
         if my_dataset.type == "text/csv":
             data = pd.read_csv(my_dataset)
@@ -1239,9 +558,9 @@ if my_dataset is not None:
         else:
             st.error("Unsupported file type. Please upload a CSV, Excel or HD5 file.")
             st.stop()
-        filtered = filter_dataframe(data)
-        st.session_state['parsed_responses'] = filtered
-        st.dataframe(filtered)
+        parser = filter_dataframe(data)
+        st.session_state['parsed_responses'] = parser
+        st.dataframe(parser)
         st.success(f"Successfully loaded and displayed data from {my_dataset.name}")
     except Exception as e:
         st.error(f"An error occurred while reading the file: {e}")
@@ -1250,85 +569,27 @@ else:
     parsed_responses = filter_dataframe(parsed)
     st.session_state['parsed_responses'] = parsed_responses
     st.dataframe(parsed_responses)
-# Gemini Q&A over a column moved to the RAG search page (rag_search.py).
+col1, col2 = st.columns(2)
+with col1:
+    col_parsed = st.selectbox("Which column do you want to query?", st.session_state['parsed_responses'].columns)
+with col2:
+    GEMINI_KEY = st.text_input('Gemini API Key', value=GEMINI_KEY, type='password', help="Enter your Gemini API key")
+
+if col_parsed and GEMINI_KEY:
+    selected_column_data = st.session_state['parsed_responses'][col_parsed].tolist()
+    question = st.text_input("Ask a question or leave empty for summarization")
+    if st.button("Generate Query") and selected_column_data:
+        st.write(gemini_query(question, selected_column_data, GEMINI_KEY))
 st.session_state['stage'] = 1
 
-# Add enhanced visualization toggle
-with st.expander("🚀 Enhanced Visualization Options", expanded=False):
-    use_interactive_viz = st.checkbox("Use Interactive Visualizations", value=True, help="Enable modern interactive charts with better performance")
-    show_data_profile = st.checkbox("Show Data Profile", value=True, help="Display intelligent data analysis before filtering")
-    enable_performance_mode = st.checkbox("Performance Mode", value=True, help="Enable smart sampling for large datasets")
-
-# TF-IDF cluster naming — default OFF. When ON, clusters get human-readable
-# names derived from their top TF-IDF terms and near-duplicate names are
-# merged. When OFF, clusters keep their numeric HDBSCAN ids and XGBoost
-# downstream runs against the raw labels.
-enable_tfidf_clusters = st.toggle(
-    "Enable TF-IDF cluster naming + merging",
-    value=False,
-    key="enable_tfidf_clusters",
-    help="When off, clusters are labeled 'Cluster N' and HDBSCAN labels are "
-         "passed straight to XGBoost. When on, top TF-IDF terms name each "
-         "cluster and near-duplicate names are merged (slower).",
-)
-
-# Categorical association explorer — runs directly on the parsed DataFrame,
-# independent of the embedding/cluster pipeline below.
-if st.session_state['stage'] > 0 and st.session_state.get('parsed_responses') is not None:
-    with st.expander("🎲 Categorical Association Explorer (Cramér's V)", expanded=False):
-        render_cramers_v_explorer(st.session_state['parsed_responses'])
-    with st.expander("🌊 Categorical Flow (Sankey)", expanded=False):
-        render_categorical_flow_sankey(st.session_state['parsed_responses'])
 
 if st.session_state['stage'] > 0 :
-    # ── High-correlation shortcut ──────────────────────────────────────────────
-    # Pre-fill the column selector with whatever the Cramér's V explorer above
-    # last flagged as strongly associated. Kept OUTSIDE the form so toggling it
-    # reruns immediately and updates the (keyless) multiselect default; a
-    # checkbox inside the form would only take effect on submit.
-    _valid_cols = list(st.session_state['parsed_responses'].columns)
-    _cv_df = st.session_state.get("cv_pass2_df")
-    if _cv_df is None:
-        _cv_df = st.session_state.get("cv_pass1_df")
-    _cv_strong = float(st.session_state.get("cv_strong_threshold", 0.30))
-    _cv_excl = bool(st.session_state.get("cv_exclude_trivial", True))
-    _high_corr_cols = [
-        c for c in high_correlation_columns(_cv_df, _cv_strong, _cv_excl)
-        if c in _valid_cols
-    ]
-
-    pass_high_corr = st.checkbox(
-        "Pass all high-correlated columns from Cramér's V",
-        value=False,
-        key="analyze_pass_high_corr",
-        disabled=not _high_corr_cols,
-        help=(
-            f"Pre-select every column reaching Cramér's V ≥ {_cv_strong:.2f} with "
-            "another column in the latest Cramér's V pass. Run the Categorical "
-            "Association Explorer above first to populate this."
-        ),
-    )
-    if not _high_corr_cols:
-        st.caption(
-            ":grey[No Cramér's V results yet — run the Categorical Association "
-            "Explorer above to enable the high-correlation shortcut.]"
-        )
-    elif pass_high_corr:
-        st.caption(
-            f":green[{len(_high_corr_cols)} high-correlation column(s) pre-selected:] "
-            f"{', '.join(_high_corr_cols[:12])}"
-            f"{'…' if len(_high_corr_cols) > 12 else ''}"
-        )
-
-    _default_cols = _high_corr_cols if (pass_high_corr and _high_corr_cols) else []
-
     with st.form(border=True, key='Select Columns for Analysis'):
         columns_to_analyze = st.multiselect(
             label='Select columns to analyze',
-            options=st.session_state['parsed_responses'].columns,
-            default=_default_cols,
+            options=st.session_state['parsed_responses'].columns
         )
-        if st.form_submit_button("Process Data"):
+        if st.form_submit_button("Process Data"):   
             if columns_to_analyze:
                 analyzers = []
                 col_names = []
@@ -1343,22 +604,11 @@ if st.session_state['stage'] > 0 :
                             analyzer.reduce_dimensionality(method='UMAP', n_components=2, n_neighbors=15, min_dist=0.1)
                             st.write("Clustering data...")
                             analyzer.cluster_data(method='HDBSCAN', min_cluster_size=15)
-                            if enable_tfidf_clusters:
-                                analyzer.get_tf_idf_clusters(top_n=3)
-                                st.write("Naming clusters...")
-                                clusters[column] = analyzer.merge_similar_clusters(
-                                    cluster_terms=analyzer.__dict__['cluster_terms'],
-                                    cluster_labels=analyzer.__dict__['cluster_labels'],
-                                )
-                            else:
-                                # TF-IDF disabled: numeric placeholder names + raw HDBSCAN labels.
-                                _labels = analyzer.__dict__['cluster_labels']
-                                analyzer.cluster_terms = pd.Categorical(
-                                    [f"Cluster {cid}" for cid in np.unique(_labels) if cid != -1]
-                                )
-                                clusters[column] = list(_labels)
+                            analyzer.get_tf_idf_clusters(top_n=3)
+                            st.write("Naming clusters...")
                             analyzers.append(analyzer)
                             col_names.append(column)
+                            clusters[column] = analyzer.merge_similar_clusters(cluster_terms=analyzer.__dict__['cluster_terms'], cluster_labels=analyzer.__dict__['cluster_labels'])
                             
                             # Run the visualization
                             # fig = datamapplot.create_plot(
@@ -1420,101 +670,42 @@ if st.session_state['stage'] > 0 :
                         st.session_state['analysis_complete'] = True
 
 
-# Gemini Q&A over processed/analyzed data moved to the RAG search page (rag_search.py).
+# this will check if the dataframe is not empty
+# if st.session_state['new_data'] is not None:
+#     parsed2 = st.session_state.get('dataset', pd.DataFrame())
+#     parsed2 = filter_dataframe(parsed2)
+#     col1, col2 = st.columns(2)
+#     st.dataframe(parsed2)
+#     with col1:
+#         col_parsed2 = st.selectbox("Which columns do you want to query?", parsed2.columns)
+#     with col2:
+#         GEMINI_KEY = st.text_input('Gemini APIs Key', GEMINI_KEY, type='password', help="Enter your Gemini API key")
+#     if col_parsed and GEMINI_KEY:
+#         selected_column_data2 = parsed2[col_parsed2].tolist()
+#         question2 = st.text_input("Ask a questions or leave empty for summarization")
+#         if st.button("Generate Query") and selected_column_data2:
+#             with st.status(f"Generating Query", expanded=True) as status:
+#                 gemini_answer = gemini_query(question2, selected_column_data2, GEMINI_KEY)
+#                 st.write(gemini_answer)
+#                 st.session_state['gemini_answer'] = gemini_answer
 
-# Enhanced visualization section
 if 'analysis_complete' in st.session_state and st.session_state['analysis_complete']:
-    st.header("📊 Enhanced Interactive Visualizations")
-    
-    # Check if enhanced visualization is enabled
-    use_interactive_viz = st.session_state.get('use_interactive_viz', True)
-    show_data_profile = st.session_state.get('show_data_profile', True)
-    enable_performance_mode = st.session_state.get('enable_performance_mode', True)
-    
-    if use_interactive_viz and 'parsed_responses' in st.session_state:
-        
-        # Get the parsed responses for visualization
-        df_for_viz = st.session_state['parsed_responses']
-        
-        # Interactive visualization options
-        viz_tab1, viz_tab2, viz_tab3 = st.tabs(["📈 Interactive Charts", "🎯 Analysis Results", "📋 Data Explorer"])
-        
-        with viz_tab1:
-            st.subheader("Interactive Data Exploration")
-            
-            # Get numeric and categorical columns
-            numeric_cols = df_for_viz.select_dtypes(include=[np.number]).columns.tolist()
-            categorical_cols = df_for_viz.select_dtypes(include=['object', 'category']).columns.tolist()
-            
-            if len(numeric_cols) >= 2:
-                col1, col2, col3 = st.columns(3)
-                
-                with col1:
-                    x_col = st.selectbox("X-axis", numeric_cols, key="enhanced_x")
-                with col2:
-                    y_col = st.selectbox("Y-axis", [col for col in numeric_cols if col != x_col], key="enhanced_y")
-                with col3:
-                    color_col = st.selectbox("Color by", ["None"] + categorical_cols, key="enhanced_color")
-                    color_col = None if color_col == "None" else color_col
-                
-                if st.button("Generate Interactive Scatter Plot", key="enhanced_scatter"):
-                    fig = Enhanced_Visualizer.plot_interactive_scatter(
-                        df_for_viz, x_col, y_col, color_col=color_col, 
-                        max_points=10000 if enable_performance_mode else len(df_for_viz)
-                    )
-                    st.plotly_chart(fig, use_container_width=True)
-            
-            # Interactive correlation matrix
-            if len(numeric_cols) >= 3:
-                st.subheader("Correlation Analysis")
-                selected_cols = st.multiselect("Select columns for correlation", numeric_cols, default=numeric_cols[:8])
-                
-                if selected_cols and len(selected_cols) >= 2:
-                    if st.button("Generate Correlation Matrix", key="enhanced_corr"):
-                        fig = Enhanced_Visualizer.plot_correlation_matrix(df_for_viz[selected_cols])
-                        st.plotly_chart(fig, use_container_width=True)
-        
-        with viz_tab2:
-            st.subheader("Analysis Results Visualization")
-            
-            # Show cluster analysis results if available
-            if st.session_state.get('clusters'):
-                for column, cluster_info in st.session_state['clusters'].items():
-                    st.write(f"### Cluster Analysis: {column}")
-                    
-                    # Create interactive treemap for clusters
-                    if len(cluster_info) > 0:
-                        cluster_df = pd.DataFrame({'cluster': cluster_info})
-                        fig = Enhanced_Visualizer.plot_interactive_treemap(cluster_df, 'cluster', top_n=15)
-                        st.plotly_chart(fig, use_container_width=True)
-        
-        with viz_tab3:
-            st.subheader("Enhanced Data Explorer")
-            
-            # Show data profile if enabled
-            if show_data_profile:
-                with st.expander("📊 Intelligent Data Profile", expanded=False):
-                    profile = DataProcessor.profile_data(df_for_viz)
-                    
-                    col1, col2, col3, col4 = st.columns(4)
-                    with col1:
-                        st.metric("Total Rows", f"{len(df_for_viz):,}")
-                    with col2:
-                        st.metric("Categorical Cols", len(profile['categorical_columns']))
-                    with col3:
-                        st.metric("Numeric Cols", len(profile['numeric_columns']))
-                    with col4:
-                        st.metric("Memory Usage", f"{profile['memory_usage'] / 1024**2:.1f} MB")
-            
-            # Enhanced filtering
-            st.write("### Advanced Data Filtering")
-            filtered_df = DataProcessor.filter_dataframe_enhanced(df_for_viz, enable_quick_filters=False, enable_advanced_filters=True)
-            
-            if len(filtered_df) != len(df_for_viz):
-                st.success(f"✅ Filtered data: {len(filtered_df):,} rows (from {len(df_for_viz):,})")
-                
-                # Option to re-run analysis on filtered data
-                if st.button("🔄 Re-run Analysis on Filtered Data"):
-                    st.session_state['parsed_responses'] = filtered_df
-                    st.session_state['analysis_complete'] = False
-                    st.rerun()
+    ticked_analysis = st.checkbox('Query Processed Data')
+    if ticked_analysis:
+        if st.session_state['new_data'] is not None:
+            parsed2 = st.session_state.get('dataset', pd.DataFrame()).copy()
+            parsed2 = filter_dataframe(parsed2)
+            col1, col2 = st.columns(2)
+            st.dataframe(parsed2)
+            with col1:
+                col_parsed2 = st.selectbox("Which columns do you want to query?", parsed2.columns)
+            with col2:
+                GEMINI_KEY = st.text_input('Gemini APIs Key', value=GEMINI_KEY, type='password', help="Enter your Gemini API key")
+            if col_parsed2 and GEMINI_KEY:
+                selected_column_data2 = parsed2[col_parsed2].tolist()
+                question2 = st.text_input("Ask a questions or leave empty for summarization")
+                if st.button("Generate Queries") and selected_column_data2:
+                    with st.status(f"Generating Query", expanded=True) as status:
+                        gemini_answer = gemini_query(question2, selected_column_data2, GEMINI_KEY)
+                        st.write(gemini_answer)
+                        st.session_state['gemini_answer'] = gemini_answer

api/main.py

@@ -1,1155 +0,0 @@
-import sys
-import os
-import io
-import time
-import logging
-import traceback
-from datetime import datetime, timezone
-import uuid
-
-import numpy as np
-import pandas as pd
-from fastapi import FastAPI, UploadFile, File, HTTPException, Query, Header, Depends
-from fastapi.middleware.cors import CORSMiddleware
-from pydantic import BaseModel
-
-# Add parent directory to path for importing uap_analyzer
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
-
-# Suppress expected outputs to only have clean API routes
-logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
-logger = logging.getLogger(__name__)
-
-app = FastAPI(title="UAP Analysis API", version="1.0.0")
-
-# CORS Improvement (Codex #6): Explicit origin allowlist, credentials disabled unless required
-origins = os.getenv("UAP_API_CORS_ORIGINS", "http://localhost:5173,http://127.0.0.1:5173")
-allow_origins = [origin.strip() for origin in origins.split(",") if origin.strip()]
-
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=allow_origins,
-    allow_credentials=False,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-# ---------------------------------------------------------------------------
-# State Management (Codex #1): Session isolation
-# ---------------------------------------------------------------------------
-class SessionState:
-    def __init__(self):
-        self.dataset = None
-        self.filtered_data = None
-        self.analyzers = []
-        self.col_names = []
-        self.new_data = None
-        self.data_processed = False
-        self.analysis_results = {}
-        self.cluster_viz = {}
-        self.cramers_v = None
-        self.analysis_runs = 0
-        self.last_analysis_at = None
-        self.analysis_mode = os.getenv("UAP_ANALYSIS_MODE", "production")  # demo vs production
-        # Parsing / SCU state (mirrors the Streamlit parsing.py session keys)
-        self.parse_source_df = None       # raw uploaded reports awaiting extraction
-        self.parsed_responses = None      # {description: parsed JSON dict}
-        self.parsed_df = None             # flat parsed-responses DataFrame
-        self.scu_normalized_df = None     # scu_normalizer.normalize() output
-        self.last_accessed = time.time()
-
-sessions = {}
-SESSION_TTL = 3600 * 24  # 24 hours
-
-def get_session(session_id: str = Header(None, alias="X-Session-ID")) -> SessionState:
-    if not session_id:
-        session_id = "default"
-    
-    # Cleanup expired sessions
-    current_time = time.time()
-    expired = [k for k, v in sessions.items() if current_time - v.last_accessed > SESSION_TTL]
-    for k in expired:
-        del sessions[k]
-        
-    if session_id not in sessions:
-        sessions[session_id] = SessionState()
-        
-    sessions[session_id].last_accessed = current_time
-    return sessions[session_id]
-
-# Dataset paths
-DATA_PATH_WEST = os.path.join(os.path.dirname(__file__), "..", "parsed_files_distance_embeds.h5")
-DATA_PATH_EAST = os.path.join(os.path.dirname(__file__), "..", "final_ufoseti_dataset.h5")
-# Default for backward compatibility
-DATA_PATH = DATA_PATH_WEST
-MAX_QUERY_CONTEXT_ROWS = int(os.getenv("UAP_QUERY_MAX_ROWS", "250"))
-MAX_QUERY_CONTEXT_CHARS = int(os.getenv("UAP_QUERY_MAX_CHARS", "24000"))
-
-# ---------------------------------------------------------------------------
-# Pydantic models
-# ---------------------------------------------------------------------------
-class FilterSpec(BaseModel):
-    column: str
-    type: str  # "categorical", "numeric", "text"
-    values: list = None
-    min_val: float = None
-    max_val: float = None
-    pattern: str = None
-
-
-class AnalysisRequest(BaseModel):
-    columns: list[str]
-    # Cluster pipeline tuning (mirrors analyzing.py controls). When enable_tfidf
-    # is False (the analyzing.py default), clusters keep numeric "Cluster N"
-    # names and raw HDBSCAN labels are passed straight to XGBoost.
-    enable_tfidf: bool = False
-    min_cluster_size: int = 15
-    n_neighbors: int = 15
-    min_dist: float = 0.1
-    top_n: int = 32
-
-
-class QueryRequest(BaseModel):
-    question: str
-    columns: list[str]
-    gemini_key: str
-
-
-# ── Parsing ────────────────────────────────────────────────────────────────
-class SchemaMergeRequest(BaseModel):
-    labels: list[str]
-    custom_fields: dict | None = None
-
-
-class SchemaCoverageRequest(BaseModel):
-    labels: list[str]
-    custom_fields: dict | None = None
-    # Dataset columns to diff against; defaults to the uploaded parse source.
-    columns: list[str] | None = None
-
-
-class ParseEstimateRequest(BaseModel):
-    columns: list[str]
-    format_json: str                   # merged schema template (JSON string)
-    model: str = "gpt-4o-mini"
-    use_cache: bool = True
-    use_batch: bool = False
-
-
-class ParseRunRequest(BaseModel):
-    columns: list[str]
-    format_json: str                   # merged schema template (JSON string)
-    provider: str = "openai"           # "openai" | "deepseek"
-    model: str = "gpt-4o-mini"
-    api_key: str
-    max_workers: int = 10
-    keep_columns: list[str] = []       # carry-through source columns
-
-
-# ── SCU ────────────────────────────────────────────────────────────────────
-class ScuFilterRequest(BaseModel):
-    criterion_keys: list[str]
-
-
-# ── RAG ────────────────────────────────────────────────────────────────────
-class RagSearchRequest(BaseModel):
-    columns: list[str]
-    question: str
-    cohere_key: str
-    top_n: int = 50
-
-
-# ── Cramér's V explorer ────────────────────────────────────────────────────
-class CramersVRequest(BaseModel):
-    columns: list[str] | None = None
-    drop_missing: bool = False
-    exclude_trivial: bool = True
-    strong_threshold: float = 0.30
-    high_threshold: int = 30
-    source: str = "dataset"            # "dataset" | "parsed"
-
-
-class ContingencyRequest(BaseModel):
-    col1: str
-    col2: str
-    drop_missing: bool = False
-    source: str = "dataset"
-
-
-class ColumnGroupsRequest(BaseModel):
-    source: str = "dataset"            # "dataset" | "parsed"
-    high_threshold: int = 30
-
-
-class XgboostRequest(BaseModel):
-    columns: list[str]
-    source: str = "dataset"            # "dataset" | "parsed"
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-def df_to_json(df: pd.DataFrame, max_rows: int = 50000, total_rows: int | None = None) -> dict:
-    df_subset = df.head(max_rows).copy()
-    for col in df_subset.columns:
-        # Convert categories and datetimes to strings
-        if df_subset[col].dtype.name == "category" or pd.api.types.is_datetime64_any_dtype(df_subset[col]):
-            df_subset[col] = df_subset[col].astype(str)
-        else:
-            # For object columns, we need to be careful with fillna if they contain dicts
-            # We use a custom fillna for objects
-            if df_subset[col].dtype == object:
-                # Fill missing values without triggering hash checks on the content
-                mask = df_subset[col].isna()
-                df_subset.loc[mask, col] = ""
-            else:
-                df_subset[col] = df_subset[col].fillna("")
-    return {
-        "columns": list(df_subset.columns),
-        "rows": df_subset.to_dict(orient="records"),
-        # total_rows reflects the full dataset; pass it explicitly when df has
-        # already been truncated upstream so the count is not misreported.
-        "total_rows": total_rows if total_rows is not None else len(df),
-        "returned_rows": len(df_subset),
-    }
-
-def get_column_stats(df: pd.DataFrame) -> list[dict]:
-    stats = []
-    for col in df.columns:
-        # Standard info
-        info = {
-            "name": col, 
-            "dtype": str(df[col].dtype), 
-            "non_null": int(df[col].notna().sum())
-        }
-        
-        # Safe nunique
-        try:
-            info["unique"] = int(df[col].nunique())
-        except (TypeError, ValueError):
-            # For unhashable types (dicts/lists), we count unique by converting to string first
-            try:
-                info["unique"] = int(df[col].astype(str).nunique())
-            except Exception:
-                info["unique"] = 0
-            
-        # Numerical stats
-        if pd.api.types.is_numeric_dtype(df[col]):
-            try:
-                if df[col].notna().any():
-                    info["min"] = float(df[col].min())
-                    info["max"] = float(df[col].max())
-                    info["mean"] = float(df[col].mean())
-                else:
-                    info["min"] = info["max"] = info["mean"] = None
-            except Exception:
-                pass
-        # Categorical / Object stats
-        elif pd.api.types.is_object_dtype(df[col]) or df[col].dtype.name == "category":
-            try:
-                top = df[col].value_counts().head(10)
-                info["top_values"] = [{"value": str(k), "count": int(v)} for k, v in top.items()]
-            except (TypeError, ValueError):
-                # Handle unhashable types in value_counts by converting to string
-                try:
-                    top = df[col].astype(str).value_counts().head(10)
-                    info["top_values"] = [{"value": str(k), "count": int(v)} for k, v in top.items()]
-                except Exception:
-                    info["top_values"] = []
-            except Exception:
-                info["top_values"] = []
-                
-        stats.append(info)
-    return stats
-
-def truncate_context(items: list[str], max_chars: int) -> tuple[str, int]:
-    context_parts = []
-    used = 0
-    chars = 0
-    for item in items:
-        to_add = item if not context_parts else f"\\n{item}"
-        if chars + len(to_add) > max_chars:
-            break
-        context_parts.append(item)
-        chars += len(to_add)
-        used += 1
-    return "\\n".join(context_parts), used
-
-# ---------------------------------------------------------------------------
-# Data endpoints
-# ---------------------------------------------------------------------------
-@app.get("/api/data/load")
-def load_data(
-    rows: int = Query(default=15000, le=50000),
-    type: str = Query(default="west"),
-    state: SessionState = Depends(get_session)
-):
-    path = DATA_PATH_EAST if type == "east" else DATA_PATH_WEST
-
-    if not os.path.exists(path):
-        raise HTTPException(status_code=404, detail=f"Dataset file not found: {os.path.basename(path)}")
-    try:
-        df = pd.read_hdf(path, key="df")
-        if "embeddings" in df.columns:
-            df = df.drop(columns=["embeddings"])
-        full_row_count = len(df)
-        df = df.head(rows)
-        state.dataset = df
-        state.filtered_data = df
-        state.data_processed = False
-        state.col_names = []
-        return {
-            "status": "ok",
-            "data": df_to_json(df, total_rows=full_row_count),
-            "column_stats": get_column_stats(df),
-        }
-    except Exception as e:
-        logger.error(traceback.format_exc())
-        raise HTTPException(status_code=500, detail=f"Error loading {type} dataset: {e}")
-
-@app.get("/api/analysis/clusters")
-def get_clusters_viz():
-    from fastapi.responses import FileResponse
-    path = os.path.join(os.path.dirname(__file__), "..", "frontend", "uap_clusters_llm.html")
-    if not os.path.exists(path):
-        # Fallback to current dir or project root
-        path = os.path.join(os.path.dirname(__file__), "..", "uap_clusters_llm.html")
-        
-    if not os.path.exists(path):
-        raise HTTPException(status_code=404, detail="Cluster visualization not found")
-    return FileResponse(path)
-
-@app.post("/api/data/upload")
-async def upload_data(file: UploadFile = File(...), state: SessionState = Depends(get_session)):
-    try:
-        contents = await file.read()
-        name = (file.filename or "").lower()
-        if name.endswith(".csv"):
-            df = pd.read_csv(io.BytesIO(contents))
-        elif name.endswith((".xlsx", ".xls")):
-            df = pd.read_excel(io.BytesIO(contents))
-        elif name.endswith(".json"):
-            import json as _json
-            obj = _json.loads(contents.decode("utf-8"))
-            df = pd.json_normalize(obj if isinstance(obj, list) else [obj])
-        else:
-            raise HTTPException(status_code=400, detail="Unsupported file type (CSV / XLSX / JSON).")
-        state.dataset = df
-        state.filtered_data = df
-        state.data_processed = False
-        return {"status": "ok", "filename": file.filename, "data": df_to_json(df), "column_stats": get_column_stats(df)}
-    except HTTPException:
-        raise
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
-
-@app.post("/api/data/filter")
-def filter_data(filters: list[FilterSpec], state: SessionState = Depends(get_session)):
-    if state.dataset is None:
-        raise HTTPException(status_code=400, detail="No dataset loaded")
-    df = state.dataset.copy()
-    try:
-        for f in filters:
-            if f.column not in df.columns:
-                continue
-            if f.type == "categorical" and f.values:
-                df = df[df[f.column].astype(str).isin([str(v) for v in f.values])]
-            elif f.type == "numeric":
-                if f.min_val is not None and f.max_val is not None:
-                    if f.min_val > f.max_val:
-                        raise ValueError(f"Invalid range: min {f.min_val} > max {f.max_val}")
-                    df = df[df[f.column].between(f.min_val, f.max_val)]
-                elif f.min_val is not None:
-                    df = df[df[f.column] >= f.min_val]
-                elif f.max_val is not None:
-                    df = df[df[f.column] <= f.max_val]
-            elif f.type == "text" and f.pattern:
-                import re
-                df = df[df[f.column].astype(str).str.contains(re.escape(f.pattern), case=False, na=False)]
-        state.filtered_data = df
-        return {"status": "ok", "data": df_to_json(df), "column_stats": get_column_stats(df)}
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Filtering failed: {e}")
-
-@app.get("/api/data/columns")
-def get_columns(state: SessionState = Depends(get_session)):
-    if state.dataset is None:
-        raise HTTPException(status_code=400, detail="No dataset loaded")
-    df = state.dataset
-    columns = []
-    for col in df.columns:
-        columns.append({
-            "name": col, "dtype": str(df[col].dtype),
-            "unique": int(df[col].nunique()), "non_null": int(df[col].notna().sum()),
-        })
-    return {"columns": columns}
-
-@app.get("/api/data/column-values")
-def get_column_values(
-    column: str = Query(...),
-    search: str = Query(default=""),
-    limit: int = Query(default=50, le=500),
-    state: SessionState = Depends(get_session),
-):
-    """Return unique values of a column, optionally filtered by a search term.
-
-    Backs the searchable categorical filter so any value can be selected,
-    not just the precomputed top-N from column statistics.
-    """
-    if state.dataset is None:
-        raise HTTPException(status_code=400, detail="No dataset loaded")
-    if column not in state.dataset.columns:
-        raise HTTPException(status_code=404, detail=f"Column '{column}' not found")
-
-    import re
-    series = state.dataset[column].astype(str)
-    counts = series.value_counts()
-    if search:
-        mask = counts.index.str.contains(re.escape(search), case=False, na=False)
-        counts = counts[mask]
-    total_matches = int(len(counts))
-    counts = counts.head(limit)
-    return {
-        "column": column,
-        "values": [{"value": str(k), "count": int(v)} for k, v in counts.items()],
-        "total_matches": total_matches,
-    }
-
-@app.post("/api/analyze/run")
-def run_analysis(req: AnalysisRequest, state: SessionState = Depends(get_session)):
-    if state.filtered_data is None or state.filtered_data.empty:
-        raise HTTPException(status_code=400, detail="No data available")
-    
-    df = state.filtered_data
-    valid_columns = [c for c in req.columns if c in df.columns]
-    if not valid_columns:
-        raise HTTPException(status_code=400, detail="No valid columns selected")
-        
-    results, cluster_viz, xgboost_results = {}, {}, {}
-    state.analyzers = []
-    new_data = pd.DataFrame()
-
-    if state.analysis_mode == "production":
-        # Codex #7: Run real embedding, UMAP, HDBSCAN cluster pipeline instead of generic mock
-        from sklearn.model_selection import train_test_split
-        from uap_analyzer import UAPAnalyzer, train_xgboost
-        
-        for col in valid_columns:
-            analyzer = UAPAnalyzer(df, column=col)
-            try:
-                analyzer.preprocess_data(top_n=req.top_n)
-                analyzer.reduce_dimensionality(
-                    method="UMAP", n_components=2,
-                    n_neighbors=req.n_neighbors, min_dist=req.min_dist,
-                )
-                analyzer.cluster_data(method="HDBSCAN", min_cluster_size=req.min_cluster_size)
-                _labels = analyzer.__dict__["cluster_labels"]
-                row_terms = None
-                if req.enable_tfidf:
-                    # TF-IDF naming + near-duplicate cluster merging. The merge
-                    # path re-encodes on GPU and is best-effort; fall back to
-                    # numeric labels if anything in it fails.
-                    try:
-                        analyzer.get_tf_idf_clusters(top_n=3)
-                        row_terms = analyzer.merge_similar_clusters(
-                            cluster_terms=analyzer.__dict__["cluster_terms"],
-                            cluster_labels=analyzer.__dict__["cluster_labels"],
-                        )
-                    except Exception as merge_err:
-                        logger.warning(f"TF-IDF naming failed for {col}, using raw labels: {merge_err}")
-                        row_terms = None
-                if row_terms is None:
-                    # Numeric placeholder names; raw HDBSCAN labels flow to XGBoost.
-                    row_terms = [f"Cluster {cid}" for cid in _labels]
-                # cluster_terms is per-row here (length == len(df)) so the
-                # cluster_viz masks below index reduced_embeddings correctly.
-                analyzer.cluster_terms = row_terms
-
-                new_data[f"Analyzer_{col}"] = analyzer.cluster_terms
-                state.analyzers.append(analyzer)
-                
-                traces = []
-                unique_terms = np.unique(analyzer.cluster_terms)
-                for term in unique_terms:
-                    mask = (np.array(analyzer.cluster_terms) == term)
-                    if not mask.any(): continue
-                    traces.append({
-                        "name": term,
-                        "x": analyzer.reduced_embeddings[mask, 0].tolist(),
-                        "y": analyzer.reduced_embeddings[mask, 1].tolist(),
-                        "text": df[col].iloc[mask].astype(str).tolist(),
-                        "count": int(mask.sum()),
-                    })
-                cluster_viz[col] = {"traces": traces, "title": f"{col} Clusters (HDBSCAN)"}
-                _dist = pd.Series(analyzer.cluster_terms).astype(str).value_counts().head(20)
-                results[col] = {
-                    "cluster_count": len(unique_terms),
-                    "total_points": len(df),
-                    "distribution": [{"label": str(k), "count": int(v)} for k, v in _dist.items()],
-                }
-            except Exception as e:
-                logger.error(f"Error analyzing {col}: {e}")
-                
-        if len(state.analyzers) >= 2:
-            new_data_cat = new_data.fillna('null').astype('category')
-            data_nums = new_data_cat.apply(lambda x: x.cat.codes)
-            for col in data_nums.columns:
-                try:
-                    categories = new_data_cat[col].cat.categories
-                    x_train, x_test, y_train, y_test = train_test_split(data_nums.drop(columns=[col]), data_nums[col], test_size=0.2, random_state=42)
-                    bst, accuracy, preds = train_xgboost(x_train, y_train, x_test, y_test, len(categories))
-                    
-                    importances_dict = bst.get_score(importance_type="gain")
-                    importances = {k.replace("Analyzer_", ""): float(v) for k, v in importances_dict.items()}
-                    importances = dict(sorted(importances.items(), key=lambda x: x[1], reverse=True))
-                    xgboost_results[col.replace("Analyzer_", "")] = {
-                        "feature_importance": importances,
-                        "accuracy": round(accuracy, 3),
-                    }
-                except Exception as e:
-                    logger.error(f"Error in xgboost for {col}: {e}")
-                    
-    else:
-        # Mock mode
-        for col in valid_columns:
-            col_data = df[col].fillna("").astype(str)
-            vc = col_data.value_counts()
-            top_labels = vc.head(32).index.tolist()
-            cmap = {l: i for i, l in enumerate(top_labels)}
-            clabels = col_data.apply(lambda x: cmap.get(x, -1)).values
-            
-            n = len(col_data)
-            np.random.seed(42)
-            red_embeds = np.random.randn(n, 2)
-            
-            cluster_terms = []
-            for i, label in enumerate(top_labels[:20]):
-                mask = col_data == label
-                red_embeds[mask.values] = (np.random.randn(2) * 3) + np.random.randn(mask.sum(), 2) * 0.5
-                
-            new_data[f"Analyzer_{col}"] = [top_labels[c] if 0 <= c < len(top_labels) else "Other" for c in clabels]
-            
-            traces = []
-            for i, term in enumerate(top_labels[:20]):
-                mask = (clabels == i)
-                if mask.sum() == 0: continue
-                traces.append({
-                    "name": term,
-                    "x": red_embeds[mask, 0].tolist(),
-                    "y": red_embeds[mask, 1].tolist(),
-                    "text": col_data[mask].tolist(),
-                    "count": int(mask.sum()),
-                })
-            cluster_viz[col] = {"traces": traces, "title": f"{col} Clusters (Mock)"}
-            results[col] = {"cluster_count": len(top_labels), "distribution": [{"label": str(k), "count": int(v)} for k, v in vc.head(20).items()], "total_points": n}
-        
-    cramers_data = None
-    if len(valid_columns) >= 2:
-        new_data_cat = new_data.fillna("null").astype("category")
-        cols = list(new_data_cat.columns)
-        from scipy.stats import chi2_contingency
-        matrix = []
-        for c1 in cols:
-            row = []
-            for c2 in cols:
-                if c1 == c2:
-                    row.append(1.0)
-                else:
-                    try:
-                        ct = pd.crosstab(new_data_cat[c1], new_data_cat[c2])
-                        chi2 = chi2_contingency(ct)[0]
-                        n_obs = ct.sum().sum()
-                        phi2 = chi2 / n_obs
-                        r, k = ct.shape
-                        phi2corr = max(0, phi2 - ((k-1)*(r-1))/(n_obs-1))
-                        rcorr, kcorr = r - ((r-1)**2)/(n_obs-1), k - ((k-1)**2)/(n_obs-1)
-                        denom = min(kcorr-1, rcorr-1)
-                        v = np.sqrt(phi2corr / denom) if denom > 0 else 0.0
-                        row.append(round(float(v), 3))
-                    except Exception: row.append(0.0)
-            matrix.append(row)
-        cramers_data = {"labels": [c.replace("Analyzer_", "") for c in cols], "matrix": matrix}
-
-    state.new_data = new_data
-    state.data_processed = True
-    state.analysis_results = results
-    state.cluster_viz = cluster_viz
-    state.cramers_v = cramers_data
-    state.col_names = valid_columns
-    state.analysis_runs += 1
-    state.last_analysis_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
-
-    return {
-        "status": "ok",
-        "analysis_mode": state.analysis_mode,
-        "mock_mode": state.analysis_mode == "mock",
-        "warnings": ["Results generated by mock pipeline for demo purposes."] if state.analysis_mode == "mock" else [],
-        "results": results,
-        "cluster_viz": cluster_viz,
-        "cramers_v": cramers_data,
-        "xgboost": xgboost_results if xgboost_results else {},
-        "processed_data": df_to_json(new_data),
-    }
-
-@app.get("/api/analyze/results")
-def get_analysis_results(state: SessionState = Depends(get_session)):
-    if not state.data_processed:
-        raise HTTPException(status_code=400, detail="No analysis run yet")
-    return {"results": state.analysis_results, "cluster_viz": state.cluster_viz, "cramers_v": state.cramers_v}
-
-@app.post("/api/query/gemini")
-def query_gemini(req: QueryRequest, state: SessionState = Depends(get_session)):
-    if state.filtered_data is None:
-        raise HTTPException(status_code=400, detail="No data")
-    valid_cols = [c for c in req.columns if c in state.filtered_data.columns]
-    if not valid_cols:
-        raise HTTPException(status_code=400, detail="No valid columns selected")
-    try:
-        import google.generativeai as genai
-        # Combine the selected columns per row with " - " (mirrors rag_search.py
-        # and the parsing _build_text_series), then drop empty rows.
-        text_series = _build_text_series(state.filtered_data, valid_cols)
-        filtered = [t for t in text_series.dropna().tolist() if t.strip()]
-        context_seed = filtered[:MAX_QUERY_CONTEXT_ROWS]
-        # Token aware chunking
-        context, rows_used = truncate_context(context_seed, MAX_QUERY_CONTEXT_CHARS)
-        if not context:
-            raise HTTPException(status_code=400, detail="No text available for querying")
-
-        genai.configure(api_key=req.gemini_key)
-        model = genai.GenerativeModel("models/gemini-3.1-pro-preview")
-        response = model.generate_content([f"{req.question or 'Summarize'}\\nContext: {context}\\n\\n"])
-        return {"status": "ok", "response": response.text,
-                "context_rows_used": rows_used, "columns_used": valid_cols}
-    except HTTPException:
-        raise
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
-
-@app.get("/api/dashboard/summary")
-def get_dashboard_summary(state: SessionState = Depends(get_session)):
-    if state.dataset is None:
-        return {"loaded": False, "analysis_runs": state.analysis_runs, "last_analysis_at": state.last_analysis_at}
-    df = state.dataset
-    return {
-        "loaded": True,
-        "total_rows": len(df),
-        "total_columns": len(df.columns),
-        "columns": list(df.columns),
-        "analyzed": state.data_processed,
-        "analyzed_columns": len(state.col_names),
-        "analysis_runs": state.analysis_runs,
-        "last_analysis_at": state.last_analysis_at,
-        "analysis_mode": state.analysis_mode,
-        "null_counts": {col: int(df[col].isna().sum()) for col in df.columns}
-    }
-
-# ---------------------------------------------------------------------------
-# Parsing — LLM feature extraction (parsing.py parity, core tier)
-# ---------------------------------------------------------------------------
-from api.services import parsing_service, scu_service, rag_service, analysis_service
-
-
-def _build_text_series(df: pd.DataFrame, columns: list[str]) -> pd.Series:
-    """Concatenate the selected raw-text columns per row with ' - ' (mirrors
-    parsing.py's _build_text), dropping empty cells."""
-    def _row(row):
-        parts = [
-            str(row[c]).strip() for c in columns
-            if c in row and pd.notna(row[c]) and str(row[c]).strip()
-        ]
-        return " - ".join(parts) if parts else None
-    return df.apply(_row, axis=1)
-
-
-@app.get("/api/parse/schemas")
-def parse_schemas():
-    try:
-        return {**parsing_service.list_schemas(), "models": parsing_service.available_models()}
-    except Exception as e:
-        logger.error(traceback.format_exc())
-        raise HTTPException(status_code=500, detail=f"Could not load schemas: {e}")
-
-
-@app.post("/api/parse/schema-merge")
-def parse_schema_merge(req: SchemaMergeRequest):
-    try:
-        return parsing_service.merge_schema(req.labels, req.custom_fields)
-    except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
-
-
-@app.post("/api/parse/schema-coverage")
-def parse_schema_coverage(req: SchemaCoverageRequest, state: SessionState = Depends(get_session)):
-    """Diff the merged schema's leaf fields against the dataset columns (🟢/🔴
-    coverage) and return per-mode extraction schemas (all / missing / database)."""
-    cols = req.columns
-    if cols is None:
-        if state.parse_source_df is None:
-            raise HTTPException(
-                status_code=400,
-                detail="No raw dataset uploaded and no columns provided.",
-            )
-        cols = list(state.parse_source_df.columns)
-    try:
-        return parsing_service.schema_coverage_report(req.labels, cols, req.custom_fields)
-    except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    except Exception as e:
-        logger.error(traceback.format_exc())
-        raise HTTPException(status_code=500, detail=f"Coverage diff failed: {e}")
-
-
-@app.post("/api/parse/upload")
-async def parse_upload(file: UploadFile = File(...), state: SessionState = Depends(get_session)):
-    """Upload a raw report dataset to be fed through the LLM extractor."""
-    try:
-        contents = await file.read()
-        name = (file.filename or "").lower()
-        if name.endswith(".csv"):
-            df = pd.read_csv(io.BytesIO(contents))
-        elif name.endswith((".xlsx", ".xls")):
-            df = pd.read_excel(io.BytesIO(contents))
-        elif name.endswith(".json"):
-            import json as _json
-            raw = contents.decode("utf-8")
-            obj = _json.loads(raw)
-            df = pd.json_normalize(obj if isinstance(obj, list) else [obj])
-        else:
-            raise HTTPException(status_code=400, detail="Unsupported file type (CSV / XLSX / JSON).")
-        state.parse_source_df = df
-        return {
-            "status": "ok",
-            "filename": file.filename,
-            "data": df_to_json(df, max_rows=200),
-            "columns": list(df.columns),
-            "total_rows": len(df),
-        }
-    except HTTPException:
-        raise
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
-
-
-@app.post("/api/parse/estimate")
-def parse_estimate(req: ParseEstimateRequest, state: SessionState = Depends(get_session)):
-    if state.parse_source_df is None:
-        raise HTTPException(status_code=400, detail="No raw dataset uploaded. Upload reports first.")
-    valid = [c for c in req.columns if c in state.parse_source_df.columns]
-    if not valid:
-        raise HTTPException(status_code=400, detail="No valid text columns selected.")
-    texts = _build_text_series(state.parse_source_df, valid).dropna().tolist()
-    if not texts:
-        raise HTTPException(status_code=400, detail="Selected columns produced no text.")
-    try:
-        return parsing_service.estimate(texts, req.format_json, req.model,
-                                        use_cache=req.use_cache, use_batch=req.use_batch)
-    except Exception as e:
-        raise HTTPException(status_code=400, detail=f"Could not estimate cost: {e}")
-
-
-@app.post("/api/parse/run")
-def parse_run(req: ParseRunRequest, state: SessionState = Depends(get_session)):
-    if state.parse_source_df is None:
-        raise HTTPException(status_code=400, detail="No raw dataset uploaded. Upload reports first.")
-    if not req.api_key:
-        raise HTTPException(status_code=400, detail="An API key is required to run extraction.")
-    src = state.parse_source_df
-    valid = [c for c in req.columns if c in src.columns]
-    if not valid:
-        raise HTTPException(status_code=400, detail="No valid text columns selected.")
-
-    text_series = _build_text_series(src, valid)
-    texts = text_series.dropna().tolist()
-    if not texts:
-        raise HTTPException(status_code=400, detail="Selected columns produced no text to parse.")
-
-    try:
-        result = parsing_service.run_parse(
-            texts, req.format_json, provider=req.provider, model=req.model,
-            api_key=req.api_key, max_workers=req.max_workers,
-        )
-    except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    except Exception as e:
-        logger.error(traceback.format_exc())
-        raise HTTPException(status_code=500, detail=f"Parsing failed: {e}")
-
-    df = result["df"]
-    # Carry-through columns: align on the raw-text input, like parsing.py.
-    if req.keep_columns and not df.empty:
-        keep = [c for c in req.keep_columns if c in src.columns]
-        if keep:
-            keep_src = src[keep].copy()
-            keep_src.index = text_series.values
-            keep_src = keep_src[keep_src.index.notna()]
-            keep_src = keep_src[~keep_src.index.duplicated(keep="first")]
-            df = df.join(keep_src, how="left")
-
-    state.parsed_responses = result["parsed_responses"]
-    state.parsed_df = df
-    return {
-        "status": "ok",
-        "n_ok": result["n_ok"],
-        "n_total": result["n_total"],
-        "n_failed": len(result["errors"]),
-        "errors": result["errors"][:10],
-        "data": df_to_json(df, max_rows=2000),
-    }
-
-
-@app.get("/api/parse/result")
-def parse_result(state: SessionState = Depends(get_session)):
-    if state.parsed_df is None:
-        raise HTTPException(status_code=400, detail="No parsed data in session.")
-    return {"status": "ok", "data": df_to_json(state.parsed_df, max_rows=2000),
-            "n_records": len(state.parsed_df)}
-
-
-# ---------------------------------------------------------------------------
-# SCU normalization (scu_normalizer parity)
-# ---------------------------------------------------------------------------
-@app.get("/api/scu/criteria")
-def scu_criteria():
-    try:
-        return scu_service.criteria_info()
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
-
-
-@app.post("/api/scu/normalize")
-def scu_normalize(state: SessionState = Depends(get_session)):
-    if not state.parsed_responses:
-        raise HTTPException(
-            status_code=400,
-            detail="No parsed responses in session. Run the Parsing step first.",
-        )
-    try:
-        result = scu_service.normalize(state.parsed_responses)
-    except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    except Exception as e:
-        logger.error(traceback.format_exc())
-        raise HTTPException(status_code=500, detail=f"Normalization failed: {e}")
-    state.scu_normalized_df = result["df"]
-    return {
-        "status": "ok",
-        "metrics": result["metrics"],
-        "audit_markdown": result["audit_markdown"],
-        "data": df_to_json(result["df"], max_rows=5000),
-    }
-
-
-@app.post("/api/scu/filter")
-def scu_filter(req: ScuFilterRequest, state: SessionState = Depends(get_session)):
-    if state.scu_normalized_df is None:
-        raise HTTPException(status_code=400, detail="No normalized data. Run SCU normalization first.")
-    try:
-        result = scu_service.filter_eligibility(state.scu_normalized_df, req.criterion_keys)
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Filter failed: {e}")
-    return {
-        "status": "ok",
-        "funnel": result["funnel"],
-        "n_passed": result["n_passed"],
-        "data": df_to_json(result["df"], max_rows=5000),
-    }
-
-
-# ---------------------------------------------------------------------------
-# RAG search — Cohere rerank (rag_search.py Dataset RAG parity)
-# ---------------------------------------------------------------------------
-@app.post("/api/rag/search")
-def rag_search(req: RagSearchRequest, state: SessionState = Depends(get_session)):
-    df = state.filtered_data if state.filtered_data is not None else state.dataset
-    if df is None or df.empty:
-        raise HTTPException(status_code=400, detail="No dataset loaded. Load data first.")
-    if not req.cohere_key:
-        raise HTTPException(status_code=400, detail="A Cohere API key is required.")
-    try:
-        result = rag_service.rerank(df, req.columns, req.question, req.cohere_key, top_n=req.top_n)
-    except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    except Exception as e:
-        logger.error(traceback.format_exc())
-        raise HTTPException(status_code=502, detail=f"Cohere rerank failed: {e}")
-    return {
-        "status": "ok",
-        "n_results": result["n_results"],
-        "searched_columns": result["searched_columns"],
-        "data": df_to_json(result["df"], max_rows=req.top_n),
-    }
-
-
-# ---------------------------------------------------------------------------
-# Cramér's V Categorical Association Explorer (analyzing.py parity)
-# ---------------------------------------------------------------------------
-def _assoc_source(state: SessionState, source: str) -> pd.DataFrame:
-    if source == "parsed":
-        if state.parsed_df is None:
-            raise HTTPException(status_code=400, detail="No parsed data in session.")
-        return state.parsed_df
-    df = state.filtered_data if state.filtered_data is not None else state.dataset
-    if df is None or df.empty:
-        raise HTTPException(status_code=400, detail="No dataset loaded.")
-    return df
-
-
-@app.post("/api/analysis/column-groups")
-def analysis_column_groups(req: ColumnGroupsRequest, state: SessionState = Depends(get_session)):
-    """Eligible categorical columns grouped by dotted parent, for the explorer's
-    group selector. Cheap (cardinality only) so it loads before any matrix compute."""
-    df = _assoc_source(state, req.source)
-    try:
-        return analysis_service.column_groups(df, high_threshold=req.high_threshold)
-    except Exception as e:
-        logger.error(traceback.format_exc())
-        raise HTTPException(status_code=500, detail=f"Column grouping failed: {e}")
-
-
-@app.post("/api/analysis/xgboost")
-def analysis_xgboost(req: XgboostRequest, state: SessionState = Depends(get_session)):
-    """XGBoost feature importance computed directly on the selected categorical
-    columns (no cluster pipeline) — fed by the Cramér's V explorer selection."""
-    df = _assoc_source(state, req.source)
-    try:
-        return analysis_service.xgboost_importance(df, req.columns)
-    except Exception as e:
-        logger.error(traceback.format_exc())
-        raise HTTPException(status_code=500, detail=f"XGBoost feature importance failed: {e}")
-
-
-@app.post("/api/analysis/cramers-v")
-def analysis_cramers_v(req: CramersVRequest, state: SessionState = Depends(get_session)):
-    df = _assoc_source(state, req.source)
-    try:
-        return analysis_service.cramers_v_report(
-            df, req.columns, drop_missing=req.drop_missing,
-            exclude_trivial=req.exclude_trivial, strong_threshold=req.strong_threshold,
-            high_threshold=req.high_threshold,
-        )
-    except Exception as e:
-        logger.error(traceback.format_exc())
-        raise HTTPException(status_code=500, detail=f"Cramér's V failed: {e}")
-
-
-@app.post("/api/analysis/contingency")
-def analysis_contingency(req: ContingencyRequest, state: SessionState = Depends(get_session)):
-    df = _assoc_source(state, req.source)
-    try:
-        return analysis_service.contingency(df, req.col1, req.col2, drop_missing=req.drop_missing)
-    except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Contingency failed: {e}")
-
-
-# ---------------------------------------------------------------------------
-# Map and Magnetic Features
-# ---------------------------------------------------------------------------
-from fastapi.responses import HTMLResponse
-from api.services.map_service import MapService
-
-@app.get("/api/map/html")
-def get_map_html(state: SessionState = Depends(get_session)):
-    if state.filtered_data is None or state.filtered_data.empty:
-        return HTMLResponse(
-            "<html><body style='background:#121212;color:white;display:flex;"
-            "justify-content:center;align-items:center;height:100vh;font-family:sans-serif;'>"
-            "<h3>No data to display. Please load and filter your dataset first.</h3>"
-            "</body></html>"
-        )
-    try:
-        html, from_cache = MapService.get_or_generate(state.filtered_data)
-        headers = {"X-Map-Cache": "HIT" if from_cache else "MISS"}
-        return HTMLResponse(content=html, headers=headers)
-    except Exception as e:
-        logger.error(f"Map Error: {traceback.format_exc()}")
-        return HTMLResponse(
-            f"<html><body style='background:#121212;color:red;'>"
-            f"<h3>Map generation failed: {e}</h3></body></html>"
-        )
-
-
-class MagneticRequest(BaseModel):
-    lat_col: str
-    lon_col: str
-    date_col: str
-    distance: int = 100
-
-def _fig_to_data_uri(fig, dpi: int = 80) -> str:
-    """Render a matplotlib figure to a base64-encoded PNG data URI on a dark background."""
-    import io, base64
-    import matplotlib.pyplot as plt
-    buf = io.BytesIO()
-    fig.savefig(buf, format="png", dpi=dpi, bbox_inches="tight", facecolor="#0d0d0d")
-    plt.close(fig)
-    buf.seek(0)
-    return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode("ascii")
-
-
-# Max events scanned per request (each match is a live BGS API call). Synchronous
-# endpoint, so this is kept small to bound request duration.
-MAGNETIC_MAX_EVENTS = 25
-MAGNETIC_TIME_BUDGET_S = 240
-
-
-@app.post("/api/magnetic/run")
-def run_magnetic(req: MagneticRequest, state: SessionState = Depends(get_session)):
-    """Cross-reference UAP events with geomagnetic observatory data.
-
-    For each event with valid coordinates and a post-1995 date, find the nearest
-    INTERMAGNET/BGS observatory within ``distance`` km, fetch its X/Y/Z/S minute
-    data around the event, and render a time-series graph. Also produces a
-    FastDTW-aligned aggregate across every matched event.
-    """
-    if state.filtered_data is None or state.filtered_data.empty:
-        raise HTTPException(status_code=400, detail="No data loaded. Load a dataset in the Data Explorer first.")
-
-    df = state.filtered_data
-    for label, col in (("Latitude", req.lat_col), ("Longitude", req.lon_col), ("Date", req.date_col)):
-        if col not in df.columns:
-            raise HTTPException(status_code=400, detail=f"{label} column '{col}' not found in dataset.")
-
-    import sys
-    proj_root = os.path.dirname(os.path.dirname(__file__))
-    if proj_root not in sys.path:
-        sys.path.insert(0, proj_root)
-    try:
-        import matplotlib
-        matplotlib.use("Agg")
-    except Exception:
-        pass
-    import magnetic
-
-    started = time.time()
-
-    # 1. Observatory list from the BGS API
-    try:
-        stations = magnetic.get_stations()
-        st_lat = pd.to_numeric(stations["Latitude"], errors="coerce").to_numpy()
-        st_lon = pd.to_numeric(stations["Longitude"], errors="coerce").to_numpy()
-    except Exception as e:
-        logger.error(f"Magnetic: station fetch failed: {traceback.format_exc()}")
-        raise HTTPException(status_code=502, detail=f"Could not reach the BGS observatory API: {e}")
-
-    # 2. Candidate events: numeric coordinates + parseable, post-1995 dates
-    work = pd.DataFrame({
-        "lat": pd.to_numeric(df[req.lat_col], errors="coerce"),
-        "lon": pd.to_numeric(df[req.lon_col], errors="coerce"),
-        "date": df[req.date_col].apply(magnetic.parse_uap_date),
-    }).dropna(subset=["lat", "lon", "date"])
-    work = work[work["lat"].between(-90, 90) & work["lon"].between(-180, 180)]
-    work = work[work["date"].dt.year >= 1995]
-    candidates = len(work)
-    if candidates == 0:
-        raise HTTPException(
-            status_code=400,
-            detail="No events with valid coordinates and a post-1995 date. Check the selected columns.",
-        )
-
-    # 3. Per-event: nearest observatory -> fetch data -> render graph
-    graphs, agg_data, agg_times = [], [], []
-    scanned = skipped_no_station = skipped_no_data = 0
-
-    for lat_, lon_, date_ in work[["lat", "lon", "date"]].itertuples(index=False):
-        if scanned >= MAGNETIC_MAX_EVENTS or (time.time() - started) > MAGNETIC_TIME_BUDGET_S:
-            break
-        scanned += 1
-
-        dists = np.array([
-            magnetic.get_haversine_distance(lat_, lon_, a, b)
-            for a, b in zip(st_lat, st_lon)
-        ])
-        nearest = int(np.nanargmin(dists))
-        if not np.isfinite(dists[nearest]) or dists[nearest] > req.distance:
-            skipped_no_station += 1
-            continue
-        station = stations.iloc[nearest]
-        iaga = str(station["IagaCode"])
-
-        d = pd.Timestamp(date_)
-        if d.tz is not None:
-            d = d.tz_localize(None)
-        start = (d - pd.Timedelta(hours=12)).strftime("%Y-%m-%d")
-        end = (d + pd.Timedelta(hours=48)).strftime("%Y-%m-%d")
-
-        try:
-            res = magnetic.get_data(iaga, start, end)
-        except Exception:
-            res = None
-        dt = (res or {}).get("datetime") or []
-        if not dt:
-            skipped_no_data += 1
-            continue
-
-        n = len(dt)
-        comp = {}
-        for k in ("X", "Y", "Z", "S"):
-            vals = res.get(k) or []
-            comp[k] = (
-                pd.to_numeric(pd.Series(vals), errors="coerce").to_numpy()
-                if len(vals) == n else np.full(n, np.nan)
-            )
-        if not any(np.isfinite(v).any() for v in comp.values()):
-            skipped_no_data += 1
-            continue
-
-        dt_naive = pd.to_datetime(pd.Series(dt), utc=True, errors="coerce").dt.tz_localize(None)
-        plotted = pd.DataFrame({"datetime": dt_naive, **comp})
-
-        subtitle = (
-            f"{d.date()}  |  {lat_:.3f}, {lon_:.3f}  |  "
-            f"{station['Name']} ({iaga})  |  {dists[nearest]:.0f} km"
-        )
-        try:
-            fig = magnetic.plot_data_custom(plotted.copy(), date=d, save_path=None, subtitle=subtitle)
-            graphs.append({
-                "title": f"{d.date()} — {station['Name']}",
-                "station": str(station["Name"]),
-                "iaga": iaga,
-                "distance_km": round(float(dists[nearest]), 1),
-                "event_date": str(d),
-                "image": _fig_to_data_uri(fig),
-            })
-        except Exception:
-            logger.error(f"Magnetic: plot failed for {iaga} {d}: {traceback.format_exc()}")
-            skipped_no_data += 1
-            continue
-
-        agg = plotted.copy()
-        agg["datetime"] = agg["datetime"].dt.tz_localize("UTC")
-        agg_data.append(agg)
-        agg_times.append(d.tz_localize("UTC"))
-
-    # 4. FastDTW-aligned aggregate across every matched event
-    aggregate = None
-    if len(agg_data) >= 2:
-        try:
-            fig = magnetic.plot_average_timeseries_with_dtw(agg_data, agg_times, window_hours=12, save_path=None)
-            aggregate = {
-                "title": f"FastDTW-aligned average across {len(agg_data)} events",
-                "image": _fig_to_data_uri(fig),
-            }
-        except Exception:
-            logger.error(f"Magnetic: DTW aggregate failed: {traceback.format_exc()}")
-
-    return {
-        "status": "ok",
-        "candidates": candidates,
-        "scanned": scanned,
-        "matched": len(graphs),
-        "skipped_no_station": skipped_no_station,
-        "skipped_no_data": skipped_no_data,
-        "distance_km": req.distance,
-        "elapsed_s": round(time.time() - started, 1),
-        "graphs": graphs,
-        "aggregate": aggregate,
-        "events": len(graphs),
-        "message": (
-            f"Scanned {scanned} of {candidates} candidate events within {req.distance} km — "
-            f"{len(graphs)} produced graphs, {skipped_no_station} had no observatory in range, "
-            f"{skipped_no_data} returned no usable data."
-        ),
-    }

api/services/__init__.py

@@ -1 +0,0 @@
-# api/services package

api/services/analysis_service.py

@@ -1,287 +0,0 @@
-"""Analysis service — the Categorical Association Explorer (Cramér's V) that runs
-directly on raw dataset columns, ported from analyzing.py. Reuses
-``uap_analyzer.cramers_v`` for the per-pair statistic.
-"""
-from __future__ import annotations
-
-import numpy as np
-import pandas as pd
-
-# Canonical label so missingness isn't fragmented into nan / None / "" etc.
-_MISSING_LABEL = "(missing)"
-_NULL_STR_TOKENS = {"nan", "none", "null", "<na>", "nat", ""}
-_CV_TOL = 1e-6  # values within this of 0 / 1 are "trivial"
-
-
-def _safe_nunique(series: pd.Series) -> int:
-    try:
-        return int(series.nunique(dropna=True))
-    except TypeError:
-        return int(series.astype(str).nunique(dropna=True))
-
-
-def band_columns(df: pd.DataFrame, high_threshold: int = 30) -> tuple[dict, dict]:
-    """Bucket columns into categorical bands by cardinality (see analyzing.py)."""
-    bands: dict[str, list[str]] = {
-        "binary": [], "low": [], "medium": [], "high": [], "constant": [],
-    }
-    nunique_map: dict[str, int] = {}
-    for c in df.columns:
-        nu = _safe_nunique(df[c])
-        nunique_map[c] = nu
-        if nu <= 1:
-            bands["constant"].append(c)
-        elif nu == 2:
-            bands["binary"].append(c)
-        elif nu <= 9:
-            bands["low"].append(c)
-        elif nu < high_threshold:
-            bands["medium"].append(c)
-        else:
-            bands["high"].append(c)
-    return bands, nunique_map
-
-
-def _eligible_categorical(bands: dict) -> list[str]:
-    """Columns the explorer scores by default: binary + low + medium cardinality
-    (high-cardinality / free-text and constant columns are unsuitable for Cramér's V)."""
-    return bands["binary"] + bands["low"] + bands["medium"]
-
-
-def group_by_parent(columns: list[str], sep: str = ".") -> list[dict]:
-    """Group nested, ``sep``-separated column names by their top-level parent
-    segment (the part before the first separator), preserving first-seen order.
-
-    e.g. ['craft.shape', 'craft.color', 'state'] ->
-        [{"parent": "craft", "columns": ["craft.shape", "craft.color"],
-          "leaves": ["shape", "color"], "nested": True},
-         {"parent": "state", "columns": ["state"], "leaves": ["state"],
-          "nested": False}]
-
-    Columns without the separator form their own single-member, non-nested group
-    so the frontend can render them as standalone chips.
-    """
-    order: list[str] = []
-    groups: dict[str, list[str]] = {}
-    for c in columns:
-        name = str(c)
-        parent = name.split(sep, 1)[0] if sep in name else name
-        if parent not in groups:
-            groups[parent] = []
-            order.append(parent)
-        groups[parent].append(c)
-    out = []
-    for parent in order:
-        members = groups[parent]
-        nested = len(members) > 1 or (sep in str(members[0]))
-        leaves = [str(m).split(sep, 1)[1] if sep in str(m) else str(m) for m in members]
-        out.append({"parent": parent, "columns": members,
-                    "leaves": leaves, "nested": nested})
-    return out
-
-
-def column_groups(df: pd.DataFrame, *, high_threshold: int = 30) -> dict:
-    """Eligible categorical columns for the explorer, grouped by dotted parent.
-
-    Cheap (only cardinality counting) so the frontend can render the parent-group
-    selector before computing the full Cramér's V matrix.
-    """
-    bands, nunique_map = band_columns(df, high_threshold=high_threshold)
-    eligible = _eligible_categorical(bands)
-    return {
-        "eligible": eligible,
-        "groups": group_by_parent(eligible),
-        "bands": bands,
-        "nunique": nunique_map,
-    }
-
-
-def _coalesce(series: pd.Series) -> pd.Series:
-    s = series.astype(str).str.strip()
-    return s.mask(s.str.lower().isin(_NULL_STR_TOKENS), _MISSING_LABEL)
-
-
-def compute_cramers_v_df(df: pd.DataFrame, cols: list[str],
-                         drop_missing: bool = False) -> pd.DataFrame:
-    from uap_analyzer import cramers_v
-
-    cv = pd.DataFrame(index=cols, columns=cols, data=np.nan, dtype=float)
-    cache = {c: _coalesce(df[c]) for c in cols}
-    for i, c1 in enumerate(cols):
-        cv.at[c1, c1] = 1.0
-        for c2 in cols[i + 1:]:
-            a, b = cache[c1], cache[c2]
-            if drop_missing:
-                keep = (a != _MISSING_LABEL) & (b != _MISSING_LABEL)
-                a, b = a[keep], b[keep]
-            v = 0.0 if len(a) == 0 else float(cramers_v(pd.crosstab(a, b)))
-            cv.at[c1, c2] = v
-            cv.at[c2, c1] = v
-    return cv
-
-
-def _is_trivial_v(v: float, tol: float = _CV_TOL) -> bool:
-    return (v <= tol) or (v >= 1.0 - tol)
-
-
-def pairs_table(cv_df: pd.DataFrame, exclude_trivial: bool = True) -> tuple[list[dict], int]:
-    rows, n_excluded = [], 0
-    cols = list(cv_df.columns)
-    for i, c1 in enumerate(cols):
-        for c2 in cols[i + 1:]:
-            v = cv_df.at[c1, c2]
-            if pd.isna(v):
-                continue
-            v = float(v)
-            if exclude_trivial and _is_trivial_v(v):
-                n_excluded += 1
-                continue
-            rows.append({"a": c1, "b": c2, "v": round(v, 3)})
-    rows.sort(key=lambda r: r["v"], reverse=True)
-    return rows, n_excluded
-
-
-def high_correlation_columns(cv_df: pd.DataFrame, strong_threshold: float = 0.30,
-                             exclude_trivial: bool = True) -> list[str]:
-    if cv_df is None or getattr(cv_df, "empty", True):
-        return []
-    out = []
-    for col in cv_df.columns:
-        others = cv_df[col].drop(labels=[col], errors="ignore")
-        for v in others:
-            if pd.isna(v):
-                continue
-            v = float(v)
-            if exclude_trivial and _is_trivial_v(v):
-                continue
-            if v >= strong_threshold:
-                out.append(col)
-                break
-    return out
-
-
-def cramers_v_report(df: pd.DataFrame, columns: list[str] | None = None, *,
-                     drop_missing: bool = False, exclude_trivial: bool = True,
-                     strong_threshold: float = 0.30, high_threshold: int = 30) -> dict:
-    """Full explorer payload: column bands, the Cramér's V matrix, the ranked
-    pair table, and the high-correlation column shortlist."""
-    bands, nunique_map = band_columns(df, high_threshold=high_threshold)
-
-    if columns:
-        cols = [c for c in columns if c in df.columns]
-    else:
-        # Default selection mirrors the explorer: binary + low + medium cardinality.
-        cols = _eligible_categorical(bands)
-
-    if len(cols) < 2:
-        return {
-            "labels": [], "matrix": [], "pairs": [], "n_excluded": 0,
-            "high_correlation_columns": [],
-            "bands": bands, "nunique": nunique_map, "selected_columns": cols,
-            "groups": group_by_parent(cols),
-        }
-
-    cv = compute_cramers_v_df(df, cols, drop_missing=drop_missing)
-    pairs, n_excluded = pairs_table(cv, exclude_trivial=exclude_trivial)
-    high = high_correlation_columns(cv, strong_threshold, exclude_trivial)
-
-    matrix = [[None if pd.isna(v) else round(float(v), 3) for v in cv.loc[r]] for r in cols]
-    return {
-        "labels": cols,
-        "matrix": matrix,
-        "pairs": pairs,
-        "n_excluded": n_excluded,
-        "high_correlation_columns": high,
-        "bands": bands,
-        "nunique": nunique_map,
-        "selected_columns": cols,
-        "groups": group_by_parent(cols),
-    }
-
-
-def contingency(df: pd.DataFrame, c1: str, c2: str, drop_missing: bool = False,
-                top_n: int = 15) -> dict:
-    """Crosstab + Cramér's V for a single pair, for the heatmap drill-down."""
-    from uap_analyzer import cramers_v
-
-    if c1 not in df.columns or c2 not in df.columns:
-        raise ValueError("Both columns must exist in the dataset.")
-    a, b = _coalesce(df[c1]), _coalesce(df[c2])
-    if drop_missing:
-        keep = (a != _MISSING_LABEL) & (b != _MISSING_LABEL)
-        a, b = a[keep], b[keep]
-    if len(a) == 0:
-        return {"row_labels": [], "col_labels": [], "matrix": [], "v": 0.0, "n": 0}
-
-    ct = pd.crosstab(a, b)
-    v = float(cramers_v(ct))
-    # Trim to the top_n most frequent categories on each axis for display.
-    row_order = ct.sum(axis=1).sort_values(ascending=False).index[:top_n]
-    col_order = ct.sum(axis=0).sort_values(ascending=False).index[:top_n]
-    ct = ct.loc[row_order, col_order]
-    return {
-        "row_labels": [str(x) for x in ct.index.tolist()],
-        "col_labels": [str(x) for x in ct.columns.tolist()],
-        "matrix": ct.values.astype(int).tolist(),
-        "v": round(v, 3),
-        "n": int(len(a)),
-    }
-
-
-# ── XGBoost feature importance on raw categorical columns ───────────────────
-# Cap on a target column's class count — XGBoost multi:softmax with hundreds of
-# classes is slow and the importances are meaningless. The explorer only feeds
-# binary/low/medium-cardinality columns, so this is just a safety net.
-_XGB_MAX_TARGET_CLASSES = 50
-
-
-def xgboost_importance(df: pd.DataFrame, columns: list[str], *,
-                       test_size: float = 0.2, random_state: int = 42) -> dict:
-    """Per-column XGBoost feature importance computed *directly* on the selected
-    raw categorical columns — predict each column from the others and report the
-    gain-based importance of every other column, plus the test accuracy.
-
-    This mirrors ``analyzing.py``'s ``analyze_and_predict`` loop but runs on the
-    raw values (the same set used by the Cramér's V explorer) instead of cluster
-    labels, so feature importance is available without the embedding/cluster
-    pipeline. Returns ``{results: {col: {feature_importance, accuracy}}, ...}``.
-    """
-    from sklearn.model_selection import train_test_split
-    from uap_analyzer import train_xgboost
-
-    cols = [c for c in columns if c in df.columns]
-    if len(cols) < 2:
-        return {
-            "results": {}, "columns": cols, "skipped": {},
-            "message": "Select at least two categorical columns for feature importance.",
-        }
-
-    # Coalesce missingness the same way Cramér's V does, then category-encode.
-    new_data = pd.DataFrame({c: _coalesce(df[c]) for c in cols}).astype("category")
-    data_nums = new_data.apply(lambda s: s.cat.codes)
-
-    results: dict[str, dict] = {}
-    skipped: dict[str, str] = {}
-    for col in cols:
-        n_classes = len(new_data[col].cat.categories)
-        if n_classes < 2:
-            skipped[col] = "constant column (one class)"
-            continue
-        if n_classes > _XGB_MAX_TARGET_CLASSES:
-            skipped[col] = f"too many classes ({n_classes}) to predict"
-            continue
-        try:
-            x = data_nums.drop(columns=[col])
-            y = data_nums[col]
-            x_train, x_test, y_train, y_test = train_test_split(
-                x, y, test_size=test_size, random_state=random_state,
-            )
-            bst, accuracy, _ = train_xgboost(x_train, y_train, x_test, y_test, n_classes)
-            # Gain-based importance; only features used in a split appear.
-            imp = {k: float(v) for k, v in bst.get_score(importance_type="gain").items()}
-            imp = dict(sorted(imp.items(), key=lambda kv: kv[1], reverse=True))
-            results[col] = {"feature_importance": imp, "accuracy": round(float(accuracy), 3)}
-        except Exception as e:  # noqa: BLE001 — one bad target shouldn't sink the rest
-            skipped[col] = str(e)
-
-    return {"results": results, "columns": cols, "skipped": skipped}

api/services/map_service.py

@@ -1,198 +0,0 @@
-"""
-map_service.py
---------------
-Encapsulates all Kepler.gl HTML generation logic.
-
-Key design decisions:
-  - Results are cached by a *data fingerprint* (hash of the DataFrame shape +
-    first/last row content).  When the user changes filters the fingerprint
-    changes, a new HTML payload is generated, and the old entry is evicted.
-  - We keep **at most `_MAX_CACHE_ENTRIES`** cached payloads to bound memory
-    use when many sessions are active.
-  - The heavy `df.copy()` that previously lived inside the endpoint is
-    eliminated: we only copy the minimal columns we actually need for the map.
-"""
-from __future__ import annotations
-
-import hashlib
-import logging
-import os
-import sys
-import traceback
-from collections import OrderedDict
-from typing import Optional
-
-import pandas as pd
-
-logger = logging.getLogger(__name__)
-
-_MAX_CACHE_ENTRIES = 20
-
-# ---------------------------------------------------------------------------
-# Internal helpers
-# ---------------------------------------------------------------------------
-
-def _fingerprint(df: pd.DataFrame) -> str:
-    """
-    Fast, deterministic fingerprint for a DataFrame.
-
-    Uses shape + a hash of a small sample (first & last 5 rows rendered as
-    CSV) so the cost is O(1) with respect to the total number of rows.
-    """
-    sample = pd.concat([df.head(5), df.tail(5)])
-    raw = f"{df.shape}|{sample.to_csv(index=False)}"
-    return hashlib.md5(raw.encode("utf-8", errors="replace")).hexdigest()
-
-
-def _build_html(df: pd.DataFrame) -> str:
-    """
-    Pure function: receive a (possibly large) DataFrame, return a Kepler.gl
-    HTML string.
-    """
-    import json
-    from keplergl import KeplerGl  # noqa: PLC0415
-
-    # DECOUPLED: Use the new lightweight utils instead of map.py
-    from api.utils.data_utils import auto_create_date_column, sanitize_dataframe_for_json  # noqa: PLC0415
-
-    # Path setup
-    base_dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
-    config_path = os.path.join(base_dir, "military_config.kgl")
-    bases_path = os.path.join(base_dir, "secret_bases.csv")
-    power_path = os.path.join(base_dir, "global_power_plant_database.csv")
-
-    kmap = KeplerGl(height=800)
-
-    # 1. Load configuration if available
-    config = None
-    if os.path.exists(config_path):
-        try:
-            with open(config_path, "r", encoding="utf-8") as f:
-                config = json.load(f)
-        except Exception as e:
-            logger.error(f"Failed to load military config: {e}")
-
-    # 2. Add Auxiliary Data (Power Plants & Secret Bases)
-    if os.path.exists(power_path):
-        try:
-            pp_df = pd.read_csv(power_path)
-            # Filter for nuclear as per config multiSelect
-            nuke_df = pp_df[pp_df["primary_fuel"] == "Nuclear"].copy()
-            nuke_df["icon"] = "control-on" # Required for icon layer
-            kmap.add_data(data=nuke_df, name="nuclear_powerplants")
-        except Exception as e:
-            logger.error(f"Failed to load power plant data: {e}")
-
-    if os.path.exists(bases_path):
-        try:
-            bases_df = pd.read_csv(bases_path)
-            bases_df["icon"] = "draw-shape"
-            kmap.add_data(data=bases_df, name="secret_bases")
-        except Exception as e:
-            logger.error(f"Failed to load secret bases: {e}")
-
-    # 3. Add user's UAP data
-    df = auto_create_date_column(df)
-    df = sanitize_dataframe_for_json(df)
-    
-    map_cols = list(df.columns)
-    lat_candidates = [c for c in map_cols if str(c).lower() in {"lat", "latitude", "city_latitude"}]
-    lon_candidates = [c for c in map_cols if str(c).lower() in {"lon", "lng", "longitude", "city_longitude"}]
-
-    if lat_candidates and lon_candidates:
-        lat_col = lat_candidates[0]
-        lon_col = lon_candidates[0]
-        needed_cols = list(set(map_cols) & set(df.columns))
-        df_map = df[needed_cols].copy()
-        df_map[lat_col] = pd.to_numeric(df_map[lat_col], errors="coerce")
-        df_map[lon_col] = pd.to_numeric(df_map[lon_col], errors="coerce")
-        df_map = df_map.dropna(subset=[lat_col, lon_col])
-
-        # 4. DYNAMIC VIEWPORT (Phase 14)
-        # Calculate center and zoom based on the current sightings data
-        if not df_map.empty:
-            lat_mean = float(df_map[lat_col].mean())
-            lon_mean = float(df_map[lon_col].mean())
-            
-            # Simple zoom heuristic based on spread
-            lat_range = df_map[lat_col].max() - df_map[lat_col].min()
-            lon_range = df_map[lon_col].max() - df_map[lon_col].min()
-            max_range = max(lat_range, lon_range)
-            
-            # log-based zoom approximation: 0 is whole world (~360), 10-12 is city
-            if max_range > 100: zoom = 2
-            elif max_range > 30: zoom = 3
-            elif max_range > 10: zoom = 4
-            elif max_range > 5: zoom = 5
-            elif max_range > 2: zoom = 6
-            elif max_range > 1: zoom = 7
-            else: zoom = 8
-            
-            if config:
-                if "mapState" not in config:
-                    config["mapState"] = {}
-                config["mapState"]["latitude"] = lat_mean
-                config["mapState"]["longitude"] = lon_mean
-                config["mapState"]["zoom"] = zoom
-                logger.info(f"Dynamically centering map at {lat_mean:.2f}, {lon_mean:.2f} (zoom={zoom})")
-
-        # Data ID aligned with military_config.kgl
-        kmap.add_data(data=df_map, name="uap_sightings")
-    else:
-        kmap.add_data(data=df, name="uap_sightings")
-
-    if config:
-        kmap.config = config
-
-    html_bytes = kmap._repr_html_()
-    return html_bytes.decode("utf-8") if isinstance(html_bytes, bytes) else html_bytes
-
-
-# ---------------------------------------------------------------------------
-# Public service
-# ---------------------------------------------------------------------------
-
-class MapService:
-    """
-    Singleton-style service that generates and caches Kepler.gl HTML payloads.
-    """
-    _cache: OrderedDict[str, str] = OrderedDict()
-
-    @classmethod
-    def get_or_generate(cls, df: pd.DataFrame) -> tuple[str, bool]:
-        """
-        Return (html_string, cache_hit).
-        """
-        # Fingerprint logic remains the same, but we could add config mtime if it changes often
-        key = _fingerprint(df)
-
-        if key in cls._cache:
-            # Promote to most-recently-used
-            cls._cache.move_to_end(key)
-            logger.info("MapService: cache HIT (key=%s…)", key[:8])
-            return cls._cache[key], True
-
-        logger.info("MapService: cache MISS — generating HTML (key=%s…)", key[:8])
-        html = _build_html(df)
-
-        # Evict oldest entry when the cache is full
-        if len(cls._cache) >= _MAX_CACHE_ENTRIES:
-            evicted = next(iter(cls._cache))
-            cls._cache.pop(evicted)
-            logger.debug("MapService: evicted cache entry %s…", evicted[:8])
-
-        cls._cache[key] = html
-        return html, False
-
-    @classmethod
-    def invalidate(cls, df: Optional[pd.DataFrame] = None) -> None:
-        """
-        Invalidate a specific entry (if *df* is provided) or the whole cache.
-        """
-        if df is None:
-            cls._cache.clear()
-            logger.info("MapService: full cache cleared")
-        else:
-            key = _fingerprint(df)
-            cls._cache.pop(key, None)
-            logger.info("MapService: cache entry %s… invalidated", key[:8])

api/services/parsing_service.py

@@ -1,312 +0,0 @@
-"""Parsing service — LLM feature extraction of raw UAP report text into
-structured JSON, mirroring the core of the Streamlit ``parsing.py`` page.
-
-Scope (core tier): schema registry + deep-merge + custom fields, cost
-estimation, and the *client-parallel* run mode (OpenAI / DeepSeek). The
-OpenAI server-batch path, SCU xlsx export and embeddings→HDF5 are intentionally
-left out of this first pass.
-
-Heavy/optional imports (``uap_analyzer``, ``config``) are pulled in lazily so
-importing this module never drags in torch or validates secrets.
-"""
-from __future__ import annotations
-
-import json
-import re
-from typing import Any, Callable
-
-
-# ── Schema registry ────────────────────────────────────────────────────────
-# Rebuilt directly from config.py (no Streamlit dependency) so it stays in
-# sync with parsing.py's SCHEMA_FORMATS / SCHEMA_FORMAT_GROUPS mapping.
-_SCHEMA_SPECS: list[tuple[str, str]] = [
-    ("SCU_v1", "FORMAT_SCU_V1"),
-    ("Default UAP Format", "FORMAT_LONG"),
-    ("SCU Spreadsheet", "FORMAT_LONG_XLSX"),
-    ("SCU_v2", "FORMAT_SCU_V2"),
-    ("SCU_v3", "FORMAT_SCU_V3"),
-    ("UFOSETI (RU)", "FORMAT_UFOSETI_RU"),
-    ("NUFORC", "FORMAT_NUFORC"),
-    ("Blue Book (USAF)", "FORMAT_BLUE_BOOK"),
-    ("UK National Archives", "FORMAT_UK_NATIONAL_ARCHIVES"),
-    ("COBEPS — PAN Notifications (BE)", "FORMAT_COBEPS_NOTIFICATIONS_PAN"),
-    ("COBEPS — COB 2021 (BE)", "FORMAT_COBEPS_COB_2021"),
-    ("GEP (DE)", "FORMAT_GEP"),
-    ("UPDB / NICAP", "FORMAT_UPDB_NICAP"),
-    ("OVNIBASE (FR)", "FORMAT_OVNIBASE"),
-    ("UFOSETI (raw)", "FORMAT_UFOSETI"),
-    ("UAP Sightings GitHub", "FORMAT_UAP_SIGHTINGS_GITHUB"),
-    ("Weinstein Pilot Catalog", "FORMAT_WEINSTEIN_PILOT_CATALOG"),
-    ("Petrowitsch LATAM", "FORMAT_PETROWITSCH_LATAM"),
-    ("UFOCAT 2023", "FORMAT_UFOCAT"),
-    ("CISU / CISUCAT (IT)", "FORMAT_CISU_CISUCAT"),
-    ("CUFOC 1977 Italy-France", "FORMAT_CUFOC_1977_ITALY_FRANCE"),
-    ("UAPCHECK Registry", "FORMAT_UAPCHECK"),
-]
-
-SCHEMA_FORMAT_GROUPS: dict[str, list[str]] = {
-    "Canonical & SCU": ["SCU_v1", "Default UAP Format", "SCU Spreadsheet", "SCU_v2", "SCU_v3"],
-    "Government & official archives": ["Blue Book (USAF)", "UK National Archives"],
-    "European national databases": [
-        "COBEPS — PAN Notifications (BE)", "COBEPS — COB 2021 (BE)",
-        "GEP (DE)", "OVNIBASE (FR)", "CISU / CISUCAT (IT)", "CUFOC 1977 Italy-France",
-    ],
-    "Research catalogs": [
-        "UPDB / NICAP", "UFOCAT 2023", "Weinstein Pilot Catalog", "Petrowitsch LATAM",
-    ],
-    "Public & crowd-sourced": ["NUFORC", "UAP Sightings GitHub", "UAPCHECK Registry"],
-    "UFOSETI / SETI": ["UFOSETI (RU)", "UFOSETI (raw)"],
-}
-
-
-def _schema_registry() -> dict[str, Any]:
-    """Map each human label to its schema (dict or JSON string), loaded from config."""
-    import config
-
-    registry: dict[str, Any] = {}
-    for label, attr in _SCHEMA_SPECS:
-        if hasattr(config, attr):
-            registry[label] = getattr(config, attr)
-    return registry
-
-
-# ── Schema merge helpers (ported from parsing.py) ──────────────────────────
-def _fmt_as_dict(v: Any) -> dict:
-    """Coerce a schema entry (dict or JSON string) to a dict; {} on failure."""
-    if isinstance(v, dict):
-        return v
-    try:
-        return json.loads(v)
-    except Exception:
-        return {}
-
-
-def _deep_merge(base: dict, override: dict) -> dict:
-    """Recursively merge ``override`` onto ``base`` without mutating either."""
-    result = base.copy()
-    for k, v in override.items():
-        if k in result and isinstance(result[k], dict) and isinstance(v, dict):
-            result[k] = _deep_merge(result[k], v)
-        else:
-            result[k] = v
-    return result
-
-
-def _flatten_dotted(d: dict, prefix: str = "") -> dict:
-    """Flatten a nested dict to {dotted_key: leaf_value}."""
-    flat: dict[str, Any] = {}
-    for k, v in d.items():
-        key = f"{prefix}.{k}" if prefix else str(k)
-        if isinstance(v, dict) and v:
-            flat.update(_flatten_dotted(v, key))
-        else:
-            flat[key] = v
-    return flat
-
-
-def list_schemas() -> dict:
-    """Public: schema labels + logical groupings for the picker."""
-    registry = _schema_registry()
-    labels = list(registry.keys())
-    groups = {g: [c for c in cols if c in registry] for g, cols in SCHEMA_FORMAT_GROUPS.items()}
-    grouped = {c for cols in groups.values() for c in cols}
-    ungrouped = [c for c in labels if c not in grouped]
-    if ungrouped:
-        groups["Other"] = ungrouped
-    return {"labels": labels, "groups": groups}
-
-
-def merge_schema(labels: list[str], custom_fields: dict | None = None) -> dict:
-    """Deep-merge the selected schema labels (plus optional dotted custom fields)
-    into a single JSON template. Returns the merged dict, its pretty JSON string,
-    the flattened leaf-field list, and the extraction key for FORMAT_LONG."""
-    registry = _schema_registry()
-    merged: dict = {}
-    for label in labels:
-        if label not in registry:
-            raise ValueError(f"Unknown schema: {label!r}")
-        merged = _deep_merge(merged, _fmt_as_dict(registry[label]))
-
-    if custom_fields:
-        merged = _deep_merge(merged, custom_fields)
-
-    top_keys = set(merged.keys())
-    # FORMAT_LONG is the only schema whose sole top-level key is sightingDetails.
-    extract_key = "sightingDetails" if top_keys == {"sightingDetails"} else None
-
-    flat = _flatten_dotted(merged)
-    fields = [
-        {"path": k, "description": v if isinstance(v, str) else json.dumps(v, ensure_ascii=False)}
-        for k, v in flat.items()
-    ]
-    return {
-        "schema": merged,
-        "schema_json": json.dumps(merged, indent=2),
-        "fields": fields,
-        "extract_key": extract_key,
-    }
-
-
-# ── Schema ↔ dataset coverage diff ─────────────────────────────────────────
-def _norm_token(s: Any) -> str:
-    """Normalize a name for fuzzy matching: lowercase, alphanumerics only."""
-    return re.sub(r"[^a-z0-9]", "", str(s).lower())
-
-
-def _leaf(path: str) -> str:
-    """Last dotted segment of a (possibly nested) field/column name."""
-    return str(path).split(".")[-1]
-
-
-def schema_coverage(merged: dict, dataset_columns: list[str]) -> dict:
-    """Compare the merged schema's leaf fields against ``dataset_columns``.
-
-    A schema field counts as *present* when some dataset column shares its
-    normalized leaf name (last dotted segment, case/punctuation-insensitive) —
-    e.g. schema ``sightingDetails.objectDescription.shape`` matches a dataset
-    column ``shape`` or ``object.shape``. Returns per-field present flags, the
-    dataset columns that matched, and the dataset-only (unmatched) columns.
-    """
-    flat = _flatten_dotted(merged)
-    col_by_token: dict[str, str] = {}
-    for c in dataset_columns:
-        col_by_token.setdefault(_norm_token(_leaf(c)), c)
-
-    coverage = []
-    matched_cols: set[str] = set()
-    for path in flat:
-        match = col_by_token.get(_norm_token(_leaf(path)))
-        if match is not None:
-            matched_cols.add(match)
-        coverage.append({
-            "path": path,
-            "leaf": _leaf(path),
-            "present": match is not None,
-            "matched_column": match,
-        })
-
-    schema_tokens = {_norm_token(_leaf(p)) for p in flat}
-    db_only = [c for c in dataset_columns if _norm_token(_leaf(c)) not in schema_tokens]
-    n_present = sum(1 for c in coverage if c["present"])
-    return {
-        "coverage": coverage,
-        "summary": {
-            "present": n_present,
-            "missing": len(coverage) - n_present,
-            "total": len(coverage),
-            "db_only": len(db_only),
-        },
-        "matched_columns": sorted(matched_cols),
-        "db_only_columns": db_only,
-    }
-
-
-def prune_schema_to_paths(merged: dict, paths: list[str]) -> dict:
-    """Rebuild a nested schema dict holding only the given dotted leaf paths,
-    preserving each leaf's original description/value and the original nesting
-    (so a FORMAT_LONG ``sightingDetails`` wrapper is kept)."""
-    flat = _flatten_dotted(merged)
-    keep = set(paths)
-    out: dict = {}
-    for path, val in flat.items():
-        if path not in keep:
-            continue
-        parts = path.split(".")
-        node = out
-        for p in parts[:-1]:
-            node = node.setdefault(p, {})
-        node[parts[-1]] = val
-    return out
-
-
-def schema_coverage_report(labels: list[str], dataset_columns: list[str],
-                           custom_fields: dict | None = None) -> dict:
-    """Coverage diff plus ready-to-use extraction-schema variants for the three
-    modes the parsing UI offers:
-
-    - ``all``      → the full merged schema (extract every field).
-    - ``missing``  → schema pruned to fields absent from the dataset (🔴 only).
-    - ``database`` → empty schema; keep the dataset columns as-is (no extraction).
-    """
-    merged = merge_schema(labels, custom_fields)["schema"]
-    cov = schema_coverage(merged, dataset_columns)
-
-    missing_paths = [c["path"] for c in cov["coverage"] if not c["present"]]
-    all_paths = [c["path"] for c in cov["coverage"]]
-    missing_schema = prune_schema_to_paths(merged, missing_paths)
-
-    variants = {
-        "all": {"schema_json": json.dumps(merged, indent=2), "n_fields": len(all_paths)},
-        "missing": {"schema_json": json.dumps(missing_schema, indent=2),
-                    "n_fields": len(missing_paths)},
-        "database": {"schema_json": "{}", "n_fields": 0},
-    }
-    return {**cov, "variants": variants}
-
-
-# ── Cost estimation ────────────────────────────────────────────────────────
-def estimate(descriptions: list[str], schema_json: str, model: str,
-             use_cache: bool = True, use_batch: bool = False) -> dict:
-    from uap_analyzer import estimate_cost
-    return estimate_cost(descriptions, schema_json, model=model,
-                         use_cache=use_cache, use_batch=use_batch)
-
-
-def available_models() -> dict:
-    from uap_analyzer import OPENAI_MODELS, DEEPSEEK_MODELS
-    return {"openai": list(OPENAI_MODELS), "deepseek": list(DEEPSEEK_MODELS)}
-
-
-# ── Parsed-response → flat DataFrame (ported from convert_cached_data_to_df) ─
-def parsed_responses_to_df(parsed_responses: dict):
-    import pandas as pd
-
-    if not parsed_responses:
-        return pd.DataFrame()
-    parsed_df_raw = pd.DataFrame(parsed_responses).T
-    if set(parsed_df_raw.columns) == {"sightingDetails"}:
-        df = pd.json_normalize(parsed_df_raw["sightingDetails"].tolist())
-        df.index = parsed_df_raw.index
-    else:
-        df = pd.json_normalize(list(parsed_responses.values()))
-        df.index = parsed_df_raw.index
-    for col in df.columns:
-        if df[col].dtype == "object":
-            df[col] = df[col].astype(str)
-    return df
-
-
-# ── Run the client-parallel parse ──────────────────────────────────────────
-def run_parse(descriptions: list[str], schema_json: str, *, provider: str,
-              model: str, api_key: str, max_workers: int = 10,
-              progress_callback: Callable[[int, int, int], None] | None = None) -> dict:
-    """Parse a list of raw report texts into structured JSON via OpenAI/DeepSeek.
-
-    Returns a dict with ``parsed_responses`` (keyed by description), the flat
-    ``df`` (pandas DataFrame), and any ``errors``.
-    """
-    from uap_analyzer import UAPParser
-
-    texts = [str(d) for d in descriptions if d is not None and str(d).strip()]
-    if not texts:
-        raise ValueError("No non-empty descriptions to parse.")
-
-    parser = UAPParser(
-        api_key=api_key,
-        model=model,
-        provider=("deepseek" if provider == "deepseek" else "openai"),
-        use_batch=False,
-        col=texts,
-    )
-    parser.process_descriptions(
-        texts, schema_json, max_workers=max_workers, progress_callback=progress_callback,
-    )
-    parsed = parser.parse_responses()
-    df = parsed_responses_to_df(parsed)
-    return {
-        "parsed_responses": parsed,
-        "df": df,
-        "errors": list(parser.last_errors),
-        "n_ok": len(parsed),
-        "n_total": len(texts),
-    }

api/services/rag_service.py

@@ -1,51 +0,0 @@
-"""RAG search service — Cohere ``rerank`` over selected columns of a dataset,
-mirroring the "Dataset RAG (Cohere)" mode of the Streamlit rag_search.py page.
-
-Each row's selected columns are serialized to a JSON document, reranked against
-the natural-language query, and returned in relevance order with scores.
-"""
-from __future__ import annotations
-
-import json
-from typing import Any
-
-# Cohere caps documents per rerank request; we chunk above this and merge.
-_RERANK_MODEL = "rerank-english-v3.0"
-
-
-def rerank(df, columns: list[str], question: str, cohere_key: str,
-           top_n: int = 50) -> dict:
-    import cohere
-    import pandas as pd  # noqa: F401  (ensures pandas present for callers)
-
-    if not columns:
-        raise ValueError("Select at least one column to search.")
-    if not question.strip():
-        raise ValueError("Enter a question to search for.")
-    missing = [c for c in columns if c not in df.columns]
-    if missing:
-        raise ValueError(f"Columns not found: {', '.join(missing)}")
-
-    documents = [
-        json.dumps(doc, default=str)
-        for doc in df[columns].to_dict("records")
-    ]
-    if not documents:
-        raise ValueError("No rows available to search.")
-
-    co = cohere.Client(api_key=cohere_key)
-    n = min(top_n, len(documents))
-    results = co.rerank(
-        model=_RERANK_MODEL,
-        query=question,
-        documents=documents,
-        top_n=n,
-        return_documents=False,
-    )
-    ranked_indices = [r.index for r in results.results]
-    ranked_scores = [float(r.relevance_score) for r in results.results]
-
-    out = df.iloc[ranked_indices].copy()
-    out.insert(0, "relevance_score", [round(s, 4) for s in ranked_scores])
-    out.insert(0, "rank", range(1, len(ranked_indices) + 1))
-    return {"df": out, "n_results": len(ranked_indices), "searched_columns": columns}

api/services/scu_service.py

@@ -1,83 +0,0 @@
-"""SCU normalization service — wraps ``scu_normalizer`` to canonicalise parsed
-UAP responses (country/US-state codes, witness roles, craft shape/size bands)
-and derive the SCU five-criterion eligibility gate, plus an eligibility filter
-with the same presets as the Streamlit page.
-"""
-from __future__ import annotations
-
-from typing import Any
-
-
-# Preset → ordered list of criterion keys, mirroring render_scu_filter() in
-# parsing.py. Built lazily so importing this module doesn't import the
-# normalizer (and its data tables) until a request actually needs it.
-def _presets(criteria_keys: list[str]) -> dict[str, list[str] | None]:
-    post_1975_keys = ["post_1975_window"] + [k for k in criteria_keys if k != "in_scu_window"]
-    return {
-        "Full five-criterion gate (1945-1975, recommended)": criteria_keys,
-        "Post-1975 five-criterion gate (1975 onwards)": post_1975_keys,
-        "Phase-3 analog — no window / anomaly / credibility gates": [
-            "has_core_fields", "has_investigation_channel",
-            "has_engagement_signal", "day_night_resolved", "military_public_known",
-        ],
-        "SCU window only (1945-1975)": ["in_scu_window"],
-        "Post-1975 window only (1975 onwards)": ["post_1975_window"],
-        "Data quality only — Criterion 2": ["has_core_fields"],
-    }
-
-
-def criteria_info() -> dict:
-    """Criterion keys/labels and named presets for the filter UI."""
-    import scu_normalizer
-
-    criteria = [
-        {"key": k, "column": c, "label": lbl} for k, c, lbl in scu_normalizer.SCU_CRITERIA
-    ]
-    extra = [
-        {"key": k, "column": c, "label": lbl} for k, c, lbl in scu_normalizer.SCU_EXTRA_CRITERIA
-    ]
-    keys = [k for k, _c, _l in scu_normalizer.SCU_CRITERIA]
-    return {"criteria": criteria, "extra_criteria": extra, "presets": _presets(keys)}
-
-
-def _raw_df_from_parsed(parsed_responses: dict):
-    import pandas as pd
-
-    return pd.json_normalize(list(parsed_responses.values()))
-
-
-def normalize(parsed_responses: dict) -> dict:
-    """Run scu_normalizer.normalize() on parsed responses.
-
-    Returns the normalized DataFrame plus the audit dict, its markdown render,
-    and headline eligibility metrics.
-    """
-    import scu_normalizer
-
-    if not parsed_responses:
-        raise ValueError("No parsed responses to normalize.")
-    raw_df = _raw_df_from_parsed(parsed_responses)
-    if raw_df.empty:
-        raise ValueError("Parsed responses produced an empty DataFrame.")
-
-    norm_df, audit = scu_normalizer.normalize(raw_df)
-    audit_md = scu_normalizer.audit_to_markdown(audit)
-
-    metrics = {
-        "rows": int(audit.get("output_rows", len(norm_df))),
-        "scu_eligible": int(audit.get("scu_eligible_count", 0)),
-        "in_scu_window": int(audit.get("in_scu_window_count", 0)),
-        "has_credible_witness": int(audit.get("has_credible_witness_count", 0)),
-    }
-    return {"df": norm_df, "audit": audit, "audit_markdown": audit_md, "metrics": metrics}
-
-
-def filter_eligibility(norm_df, criterion_keys: list[str]) -> dict:
-    """AND the selected SCU criterion columns and return the surviving rows plus
-    a cumulative funnel (one stage per criterion)."""
-    import scu_normalizer
-
-    stages, values, mask = scu_normalizer.incremental_funnel(norm_df, criterion_keys)
-    filtered = norm_df[mask.values] if len(mask) == len(norm_df) else norm_df[mask]
-    funnel = [{"stage": s, "count": v} for s, v in zip(stages, values)]
-    return {"df": filtered, "funnel": funnel, "n_passed": int(mask.sum())}

api/utils/data_utils.py

@@ -1,186 +0,0 @@
-import pandas as pd
-import numpy as np
-import datetime
-
-def sanitize_dataframe_for_json(df: pd.DataFrame) -> pd.DataFrame:
-    """Convert non-JSON-serializable values (datetime, date, time, period, timedelta) to strings."""
-    df_safe = df.copy()
-
-    # Ensure simple integer index and string column names
-    df_safe = df_safe.reset_index(drop=True)
-    df_safe.columns = df_safe.columns.map(str)
-
-    # Datetime columns: export as ISO 8601 strings with millisecond precision, null-safe
-    for col in df_safe.select_dtypes(include=["datetime64[ns]"]).columns:
-        try:
-            dt = pd.to_datetime(df_safe[col], errors='coerce')
-            # format to ISO with milliseconds (slice microseconds to 3 digits)
-            iso = dt.dt.strftime('%Y-%m-%dT%H:%M:%S.%f').str.slice(0, 23)
-            # set None where NaT
-            iso = iso.where(dt.notna(), None)
-            df_safe[col] = iso
-        except Exception:
-            # Fallback to plain string
-            df_safe[col] = df_safe[col].astype(object).where(~df_safe[col].isna(), None)
-
-    # Object columns: convert date/time-like objects to strings
-    def _to_serializable(x):
-        try:
-            # Safe null check using a list contact for unhashable types
-            if x is None or (not isinstance(x, (dict, list)) and pd.isna(x)):
-                return None
-        except Exception:
-            # Fallback if the check itself fails
-            if x is None:
-                return None
-            
-        if isinstance(x, pd.Timestamp):
-            try:
-                return x.strftime('%Y-%m-%d %H:%M:%S')
-            except Exception:
-                return None
-        if isinstance(x, (datetime.datetime, datetime.date)):
-            return x.isoformat()
-        if isinstance(x, datetime.time):
-            return x.strftime('%H:%M:%S')
-        # If it's a dict or list, it's already serializable by json.dumps if types match
-        # but we might want to ensure it's clean. For now we leave it for the JSON serializer.
-        return x
-
-    for col in df_safe.select_dtypes(include=['object']).columns:
-        try:
-            # Try efficient mapping
-            df_safe[col] = df_safe[col].map(_to_serializable)
-        except (TypeError, Exception):
-            # Fallback to slower apply or manual loop
-            try:
-                df_safe[col] = df_safe[col].apply(_to_serializable)
-            except Exception:
-                # Last resort: convert everything to string if mapping fails entirely
-                df_safe[col] = df_safe[col].astype(str).where(df_safe[col].notna(), None)
-
-    return df_safe
-
-def detect_and_combine_date_columns(df: pd.DataFrame) -> pd.DataFrame:
-    """
-    Detect separate year, month, day, hour columns and combine them into datetime columns.
-    Removed streamlit logging for API safety.
-    """
-    df_combined = df.copy()
-    
-    year_cols = [col for col in df.columns if any(pattern in col.lower() for pattern in ['year', 'yr', 'yyyy'])]
-    month_cols = [col for col in df.columns if any(pattern in col.lower() for pattern in ['month', 'mon', 'mm'])]
-    day_cols = [col for col in df.columns if any(pattern in col.lower() for pattern in ['day', 'dd', 'date'])]
-    hour_cols = [col for col in df.columns if any(pattern in col.lower() for pattern in ['hour', 'hr', 'hh', 'time'])]
-    
-    date_combinations = []
-    
-    for year_col in year_cols:
-        for month_col in month_cols:
-            for day_col in day_cols:
-                try:
-                    sample_size = min(10, len(df))
-                    test_sample = df.iloc[:sample_size]
-                    
-                    years = pd.to_numeric(test_sample[year_col], errors='coerce')
-                    months = pd.to_numeric(test_sample[month_col], errors='coerce')
-                    days = pd.to_numeric(test_sample[day_col], errors='coerce')
-                    
-                    valid_years = years.between(1900, 2100).all()
-                    valid_months = months.between(1, 12).all()
-                    valid_days = days.between(1, 31).all()
-                    
-                    if valid_years and valid_months and valid_days:
-                        combination = {'year': year_col, 'month': month_col, 'day': day_col, 'hour': None}
-                        for hour_col in hour_cols:
-                            try:
-                                hours = pd.to_numeric(test_sample[hour_col], errors='coerce')
-                                if hours.between(0, 23).all():
-                                    combination['hour'] = hour_col
-                                    break
-                            except:
-                                continue
-                        date_combinations.append(combination)
-                except Exception:
-                    continue
-    
-    for i, combo in enumerate(date_combinations):
-        try:
-            datetime_col_name = f"datetime_combined_{i+1}"
-            year = pd.to_numeric(df_combined[combo['year']], errors='coerce')
-            month = pd.to_numeric(df_combined[combo['month']], errors='coerce')
-            day = pd.to_numeric(df_combined[combo['day']], errors='coerce')
-            
-            if combo['hour']:
-                hour = pd.to_numeric(df_combined[combo['hour']], errors='coerce')
-                df_combined[datetime_col_name] = pd.to_datetime({'year': year, 'month': month, 'day': day, 'hour': hour}, errors='coerce')
-            else:
-                df_combined[datetime_col_name] = pd.to_datetime({'year': year, 'month': month, 'day': day}, errors='coerce')
-        except Exception:
-            continue
-    
-    return df_combined
-
-def auto_create_date_column(df: pd.DataFrame) -> pd.DataFrame:
-    """Automatically create a unified datetime column `date_x` for mapping."""
-    df_auto = df.copy()
-
-    if 'date_x' in df_auto.columns:
-        try:
-            parsed = pd.to_datetime(df_auto['date_x'], errors='coerce')
-            if parsed.notna().sum() > 0:
-                df_auto['date_x'] = parsed
-                return df_auto
-        except Exception:
-            pass
-
-    candidate_cols = [c for c in df_auto.columns if any(k in c.lower() for k in ['date', 'datetime', 'timestamp'])]
-    candidate_cols.extend([c for c in df_auto.select_dtypes(include=["datetime64[ns]"]).columns if c not in candidate_cols])
-    
-    best_col = None
-    best_valid = -1
-    for col in candidate_cols:
-        try:
-            parsed = pd.to_datetime(df_auto[col], errors='coerce')
-            valid = parsed.notna().sum()
-            if valid > best_valid:
-                best_valid = valid
-                best_col = col
-        except Exception:
-            continue
-
-    if best_col is not None and best_valid > 0:
-        try:
-            df_auto['date_x'] = pd.to_datetime(df_auto[best_col], errors='coerce')
-            try:
-                df_auto['date_x'] = df_auto['date_x'].dt.tz_localize(None)
-            except Exception:
-                pass
-            df_auto['date_x'] = df_auto['date_x'].dt.floor('ms')
-            return df_auto
-        except Exception:
-            pass
-
-    df_combined = detect_and_combine_date_columns(df_auto)
-    combined_cols = [c for c in df_combined.columns if c.startswith('datetime_combined_')]
-    best_combined = None
-    best_combined_valid = -1
-    for col in combined_cols:
-        try:
-            valid = df_combined[col].notna().sum()
-            if valid > best_combined_valid:
-                best_combined_valid = valid
-                best_combined = col
-        except Exception:
-            continue
-
-    if best_combined is not None and best_combined_valid > 0:
-        df_combined['date_x'] = pd.to_datetime(df_combined[best_combined], errors='coerce')
-        try:
-            df_combined['date_x'] = df_combined['date_x'].dt.tz_localize(None)
-        except Exception:
-            pass
-        df_combined['date_x'] = df_combined['date_x'].dt.floor('ms')
-        return df_combined
-
-    return df_auto

app.py

@@ -1,4 +1,108 @@
 import streamlit as st
+from st_paywall import add_auth
+from st_paywall.google_auth import get_logged_in_user_email, show_login_button
+from st_paywall.stripe_auth import is_active_subscriber, redirect_button
+from st_paywall.buymeacoffee_auth import get_bmac_payers
+import st_paywall.google_auth as google_auth
+import st_paywall.stripe_auth as stripe_auth
+
+payment_provider = st.secrets.get("payment_provider", "stripe")
+
+def add_auth(
+    required: bool = True,
+    login_button_text: str = "Login with Google",
+    login_button_color: str = "#FD504D",
+    login_sidebar: bool = True,
+    subscribe_color_button = "#FFA500",
+
+):
+    if required:
+        require_auth(
+            login_button_text=login_button_text,
+            login_sidebar=login_sidebar,
+            login_button_color=login_button_color,
+        )
+    else:
+        optional_auth(
+            login_button_text=login_button_text,
+            login_sidebar=login_sidebar,
+            login_button_color=login_button_color,
+        )
+
+def require_auth(
+    login_button_text: str = "Login with Google",
+    login_button_color: str = "#FD504D",
+    subscribe_button_color: str = "#FFA500",
+    login_sidebar: bool = True,):
+    
+    user_email = get_logged_in_user_email()
+
+    if not user_email:
+        show_login_button(
+            text=login_button_text, color=login_button_color, sidebar=login_sidebar
+        )
+        st.stop()
+    if payment_provider == "stripe":
+        is_subscriber = user_email and is_active_subscriber(user_email)
+    elif payment_provider == "bmac":
+        is_subscriber = user_email and user_email in get_bmac_payers()
+    else:
+        raise ValueError("payment_provider must be 'stripe' or 'bmac'")
+
+    if not is_subscriber:
+        redirect_button(
+            text="Make a Donation",
+            customer_email=user_email,
+            payment_provider=payment_provider,
+        )
+        st.session_state.user_subscribed = False
+        st.stop()
+    elif is_subscriber:
+        st.session_state.user_subscribed = True
+
+    if st.sidebar.button("Logout", type="primary"):
+        del st.session_state.email
+        del st.session_state.user_subscribed
+        st.rerun()
+
+def optional_auth(
+    login_button_text: str = "Login with Google",
+    login_button_color: str = "#FD504D",
+    login_sidebar: bool = True,
+    subscribe_button_color: str = "#FFA500",  # Add this line
+):
+    user_email = get_logged_in_user_email()
+    if payment_provider == "stripe":
+        is_subscriber = user_email and is_active_subscriber(user_email)
+    elif payment_provider == "bmac":
+        is_subscriber = user_email and user_email in get_bmac_payers()
+    else:
+        raise ValueError("payment_provider must be 'stripe' or 'bmac'")
+
+    if not user_email:
+        show_login_button(
+            text=login_button_text, color=login_button_color, sidebar=login_sidebar
+        )
+        st.session_state.email = ""
+        st.sidebar.markdown("")
+
+    if not is_subscriber:
+        redirect_button(
+            text="Subscribe now!", customer_email="", payment_provider=payment_provider, color=subscribe_button_color
+        )
+        st.sidebar.markdown("")
+        st.session_state.user_subscribed = False
+
+    elif is_subscriber:
+        st.session_state.user_subscribed = True
+
+    if st.session_state.email != "":
+        if st.sidebar.button("Logout", type="primary"):
+            del st.session_state.email
+            del st.session_state.user_subscribed
+            st.rerun()
+
+
 
 st.set_page_config(
    page_title="UAP Analytics",
@@ -7,6 +111,15 @@ st.set_page_config(
    initial_sidebar_state="expanded",
 )
 
+add_auth(required=False, login_button_color="#FFA500",subscribe_color_button="#FFA500")
+
+if st.session_state.email is not '':
+    st.write('')
+    st.write(f'User: {st.session_state.email}')
+    
+if "buttons" in st.session_state:
+   st.session_state.buttons = st.session_state.buttons
+
 from PIL import Image
 import base64
 
@@ -28,30 +141,14 @@ def set_png_as_page_bg(png_file):
     st.markdown(page_bg_img, unsafe_allow_html=True)
 
 
-# if st.toggle('Set background image', True):
-#     set_png_as_page_bg('saucer.webp')  # Replace with your background image path
-
-# Global pipeline option — read by parsing.py to optionally post-process
-# parsed UAP data through the SCU v2 normalizer.
-st.sidebar.toggle(
-    'Apply SCU normalization to parsed data',
-    value=False,
-    key='scu_normalize_enabled',
-    help=(
-        'When enabled, the UAP Feature Extraction page runs the SCU v2 '
-        'normalizer on parsed results — canonicalising countries, states, '
-        'witness roles and craft fields, and deriving the SCU five-criterion '
-        'eligibility gate. Adds a normalized CSV and audit report to download.'
-    ),
-)
+if st.toggle('Set background image', True):
+    set_png_as_page_bg('saucer.webp')  # Replace with your background image path
 
 pg = st.navigation([
-            st.Page("preprocessing.py", title="Document Preprocessing (Scrape → OCR → Reports → Table)", icon="🧪"),
             st.Page("rag_search.py", title="Smart-Search (Retrieval Augmented Generations)", icon="🔍"),
             st.Page("parsing.py", title="UAP Feature Extraction (Shape, Speed, Color)", icon="📄"),
             st.Page("analyzing.py", title="Statistical Analysis (UMAP+HDBSCAN, XGBoost, V-Cramer)", icon="🧠"),
             st.Page("magnetic.py", title="Magnetic Anomaly Detection (InterMagnet Stations)", icon="🧲"),
             st.Page("map.py", title="Interactive Map (Tracking variations, Proximity with Military Bases, Nuclear Facilities)", icon="🗺️"),
-            st.Page("pdf_ocr.py", title="PDF OCR — Table Extractor (LightOnOCR)", icon="📑"),
         ])
 pg.run()

app2.py

@@ -6,7 +6,9 @@ import umap
 import plotly.graph_objects as go
 from sentence_transformers import SentenceTransformer
 import torch
-from uap_analyzer import get_embed_model
+with torch.no_grad():
+    embed_model = SentenceTransformer('embaas/sentence-transformers-e5-large-v2')
+    embed_model.to('cuda')
 from sentence_transformers.util import pytorch_cos_sim, pairwise_cos_sim
 #from stqdm.notebook import stqdm
 #stqdm.pandas()
@@ -53,7 +55,8 @@ import os
 import time
 import concurrent.futures
 from requests.exceptions import HTTPError
-from tqdm import tqdm
+from stqdm import stqdm
+stqdm.pandas()
 import json
 import pandas as pd
 from openai import OpenAI
@@ -116,7 +119,8 @@ class UAPAnalyzer:
         self.y_test = None
         self.preds = None
         self.new_dataset = None
-        self.model = None  # lazy-loaded via get_embed_model() on first use
+        self.model = SentenceTransformer('embaas/sentence-transformers-e5-large-v2')
+        self.model = self.model.to('cuda')
         #self.cluster_names_ = pd.DataFrame()
 
         logging.info("UAPAnalyzer initialized")
@@ -160,7 +164,7 @@ class UAPAnalyzer:
         """
         logging.info("Extracting embeddings")
         # convert to str
-        return get_embed_model().encode(data_column.tolist(), show_progress_bar=True)
+        return embed_model.encode(data_column.tolist(), show_progress_bar=True)
 
     @spaces.GPU
     def reduce_dimensionality(self, method='UMAP', n_components=2, **kwargs):
@@ -283,7 +287,7 @@ class UAPAnalyzer:
                         merge_mapping[name1].add(name2)
 
         elif distance == 'cosine':
-            self.cluster_terms_embeddings = get_embed_model().encode(self.cluster_terms)
+            self.cluster_terms_embeddings = embed_model.encode(self.cluster_terms)
             cos_sim_matrix = pytorch_cos_sim(self.cluster_terms_embeddings, self.cluster_terms_embeddings)
             for i, name1 in enumerate(self.cluster_terms):
                 for j, name2 in enumerate(self.cluster_terms[i + 1:], start=i + 1):
@@ -343,7 +347,7 @@ class UAPAnalyzer:
 
         elif distance == 'cosine':
             if self.cluster_terms_embeddings is None:
-                self.cluster_terms_embeddings = get_embed_model().encode(self.cluster_terms)
+                self.cluster_terms_embeddings = embed_model.encode(self.cluster_terms)
             cos_sim_matrix = pytorch_cos_sim(self.cluster_terms_embeddings, self.cluster_terms_embeddings)
             for i in range(len(self.cluster_terms)):
                 for j in range(i + 1, len(self.cluster_terms)):
@@ -391,7 +395,7 @@ class UAPAnalyzer:
     def cluster_cosine(self, cluster_terms, cluster_labels, similarity_threshold):
         from sklearn.metrics.pairwise import cosine_similarity
 
-        cluster_terms_embeddings = get_embed_model().encode(cluster_terms)
+        cluster_terms_embeddings = embed_model.encode(cluster_terms)
         # Compute cosine similarity matrix in a vectorized form
         cos_sim_matrix = cosine_similarity(cluster_terms_embeddings, cluster_terms_embeddings)
 
@@ -968,7 +972,7 @@ class UAPParser:
         with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
             future_to_desc = {executor.submit(self.fetch_response, desc, format_long): desc for desc in descriptions}
 
-            for future in tqdm(concurrent.futures.as_completed(future_to_desc), total=len(descriptions)):
+            for future in stqdm(concurrent.futures.as_completed(future_to_desc), total=len(descriptions)):
                 desc = future_to_desc[future]
                 try:
                     response = future.result()
@@ -1016,12 +1020,13 @@ import seaborn as sns
 from Levenshtein import distance
 from sklearn.model_selection import train_test_split
 from sklearn.metrics import confusion_matrix
-from tqdm import tqdm
+from stqdm import stqdm
+stqdm.pandas()
 import streamlit.components.v1 as components
 from dateutil import parser
 from sentence_transformers import SentenceTransformer
 import torch
-# st.set_option('deprecation.showPyplotGlobalUse', False)
+st.set_option('deprecation.showPyplotGlobalUse', False)
 
 
 from pandas.api.types import (
@@ -1063,7 +1068,7 @@ def gemini_query(question, selected_data, gemini_key):
     context = '\n'.join(filtered)
 
     genai.configure(api_key=gemini_key)
-    query_model = genai.GenerativeModel('models/gemini-3.1-pro-preview')
+    query_model = genai.GenerativeModel('models/gemini-1.5-pro-latest')
     response = query_model.generate_content([f"{question}\n Answer based on this context: {context}\n\n"])
     return(response.text)
 

claude_app_analysis.md

@@ -1,178 +0,0 @@
-# UAP Analysis Tool — Improvement & Debugging Report
-
-## 1. Critical Bugs
-
-### 1.1 Variable Name Typo in Clustering (`uap_analyzer.py:316`)
-`self.clusters_labels` is assigned instead of `self.cluster_labels`, creating a new attribute and silently breaking downstream cluster references.
-
-### 1.2 Bare `except:` Clauses (`uap_analyzer.py:987-990, 792`)
-Multiple bare `except:` blocks swallow all exceptions including `KeyboardInterrupt` and `SystemExit`. These should catch specific exceptions (`json.JSONDecodeError`, `ValueError`, etc.).
-
-### 1.3 Race Condition in Concurrent Parsing (`uap_analyzer.py:966-978`)
-`ThreadPoolExecutor(max_workers=32)` writes to a shared `self.responses` dict without any locking mechanism. This can corrupt data under load.
-
-### 1.4 Division by Zero in Cramér's V (`uap_analyzer.py:758-761`)
-```python
-return np.sqrt(phi2corr / min((k_corr-1), (r_corr-1)))
-```
-If either dimension is 1, the denominator is 0. No guard exists.
-
----
-
-## 2. Error Handling Gaps
-
-| Area | Issue |
-|------|-------|
-| **GPU fallback** | No `try/except` around CUDA operations; crashes if GPU unavailable |
-| **Empty DataFrames** | No validation before passing to UMAP, HDBSCAN, or XGBoost |
-| **Single-valued columns** | Clustering and correlation break on columns with only one unique value |
-| **API timeouts** | No connection/read timeouts on Gemini, OpenAI, or INTERMAGNET calls |
-| **HDF5 loading** | Backend assumes the HDF5 file exists at a fixed path with no fallback |
-| **File uploads** | No size limits enforced; no validation of CSV/Excel structure |
-| **Regex injection** | User text input passed directly to `str.contains()` without `re.escape()` |
-
----
-
-## 3. Architecture & Code Quality
-
-### 3.1 God Object Anti-Pattern
-`UAPAnalyzer` handles embedding, dimensionality reduction, clustering, TF-IDF naming, XGBoost classification, and Cramér's V correlation all in one class. Consider splitting into:
-- `EmbeddingService`
-- `ClusteringPipeline`
-- `StatisticalAnalyzer`
-- `FeatureImportanceAnalyzer`
-
-### 3.2 Duplicate Methods
-`merge_similar_clusters()` and `merge_similar_clusters2()` (lines 263-356) are near-identical. Consolidate into a single parameterized method.
-
-### 3.3 In-Memory State Management (backend/main.py)
-The backend stores all session state in a plain Python dict. This means:
-- No persistence across server restarts
-- No multi-user isolation
-- Memory grows unbounded with concurrent sessions
-
-### 3.4 Hardcoded Data Truncation (`streamlit_uap_clean.py:239`)
-`.head(10000)` silently drops rows beyond 10k. Users are not warned about data loss.
-
-### 3.5 Wildcard CORS (`backend/main.py:30`)
-`allow_origins=["*"]` allows any domain to call the API. Should be restricted to the frontend origin.
-
----
-
-## 4. Data Analysis Feature Improvements
-
-### 4.1 Temporal Analysis
-- **Time-series decomposition**: Detect seasonal and trend components in sighting frequency over time (e.g., monthly/yearly cycles).
-- **Change-point detection**: Identify statistically significant shifts in sighting patterns using algorithms like PELT or Bayesian Online Change Point Detection.
-- **Temporal clustering**: Group sightings by time windows and compare feature distributions across eras.
-
-### 4.2 Enhanced Geospatial Analysis
-- **Spatial autocorrelation** (Moran's I): Quantify whether sightings cluster geographically beyond random chance.
-- **Kernel density estimation**: Generate continuous heatmaps instead of discrete point maps.
-- **Proximity analysis**: Correlate sighting density with distance to military bases, nuclear plants, airports, and flight corridors.
-- **Voronoi tessellation**: Partition geography into regions of influence per cluster.
-
-### 4.3 Advanced Clustering
-- **Silhouette score / Davies-Bouldin index**: Automatically evaluate cluster quality and suggest optimal `min_cluster_size`.
-- **Hierarchical HDBSCAN tree**: Expose the cluster hierarchy for interactive drill-down.
-- **Ensemble clustering**: Combine HDBSCAN + KMeans + spectral clustering via consensus for more robust assignments.
-- **Outlier analysis**: Surface and profile noise points (HDBSCAN label `-1`) instead of discarding them.
-
-### 4.4 Natural Language & Text Mining
-- **Topic modeling** (BERTopic / LDA): Extract latent themes from witness descriptions beyond TF-IDF keywords.
-- **Sentiment analysis**: Score witness reports for emotional intensity, fear, certainty, etc.
-- **Named entity extraction**: Pull out specific aircraft types, locations, agencies, and dates from free text.
-- **Cross-report similarity network**: Build a graph of similar reports and detect communities.
-
-### 4.5 Statistical Rigor
-- **Multiple hypothesis correction**: Apply Bonferroni or FDR correction to chi-squared tests across many column pairs.
-- **Effect size reporting**: Report Cohen's w alongside p-values for contingency tests.
-- **Confidence intervals**: Add bootstrap CIs to feature importance scores and cluster statistics.
-- **Bayesian alternatives**: Offer Bayesian correlation and classification as alternatives to frequentist methods.
-
-### 4.6 Interactive Exploration
-- **Linked brushing**: Selecting points on a scatter plot should filter the data table, map, and histograms simultaneously.
-- **Drill-down from clusters**: Click a cluster to view its members, top features, and representative reports.
-- **Comparison mode**: Side-by-side analysis of two clusters or two time periods.
-- **Custom derived columns**: Let users create calculated fields (e.g., `duration_minutes / distance_km`).
-
-### 4.7 Export & Reporting
-- **PDF/HTML report generation**: One-click export of the full analysis pipeline with charts and summary text.
-- **Reproducibility logs**: Record all parameter choices (UMAP neighbors, cluster size, etc.) so analyses can be replicated.
-- **Data export**: Export filtered/clustered data as CSV with cluster labels and embeddings.
-
----
-
-## 5. Performance Improvements
-
-| Bottleneck | Current | Suggested |
-|------------|---------|-----------|
-| Embedding computation | CPU fallback is slow for >5k rows | Batch with `encode(batch_size=256)`, cache embeddings to disk |
-| UMAP on large datasets | O(n log n), no progress feedback | Use `umap.parametric_umap` or pre-reduce with PCA to 50 dims first |
-| XGBoost training | Single-threaded default | Set `nthread=-1`, use `early_stopping_rounds` |
-| TF-IDF vectorization | Rebuilds on every run | Cache vectorizer and matrix in session state |
-| HDF5 loading | Loads full 1.8GB file into memory | Use `pd.read_hdf()` with `where` clause for lazy loading |
-| Frontend re-renders | Full data sent on every filter | Implement server-side pagination and send only visible rows |
-
----
-
-## 6. Testing (Currently Zero Coverage)
-
-### Priority Test Targets
-1. **Clustering pipeline**: empty input, single row, all-null columns, single-valued features
-2. **API endpoints**: request validation, error responses, concurrent requests
-3. **Statistical functions**: known-answer tests for Cramér's V, chi-squared, feature importance
-4. **Data loading**: corrupted files, missing columns, encoding issues, oversized uploads
-5. **Frontend components**: render tests, API error states, filter interactions
-
-### Suggested Setup
-- `pytest` + `pytest-cov` for backend
-- `vitest` + `@testing-library/react` for frontend
-- CI pipeline via GitHub Actions
-
----
-
-## 7. Security
-
-- **API keys**: Entered via text input and stored in session state in plaintext. Use environment variables or a secrets manager.
-- **CORS**: Wildcard `*` origin should be replaced with explicit frontend URL.
-- **Input sanitization**: User-provided regex and column names should be escaped before use in queries.
-- **Rate limiting**: No rate limiting on API endpoints; vulnerable to abuse.
-- **Dependency pinning**: All requirements use `>=` with no upper bounds, risking breaking changes on install.
-
----
-
-## 8. Dependency Cleanup
-
-| Package | Status |
-|---------|--------|
-| `st-paywall>=0.1.8` | Unused — remove |
-| `cohere>=5.5.8` | Imported but never called — remove or integrate |
-| `protobuf>=4.25.3` | Transitive dependency conflict risk — pin version |
-| `sentence_transformers` | Two different models loaded (`all-mpnet-base-v2` and `e5-large-v2`) — standardize on one |
-
----
-
-## 9. Logging & Observability
-
-- Replace all `print()` statements with Python `logging` module.
-- Add structured logging with context (user session, operation, duration).
-- Instrument key operations (embedding time, clustering time, API latency) with timing metrics.
-- Add health check endpoint that validates dependencies (GPU, model files, HDF5 availability).
-
----
-
-## Summary: Top 10 Action Items
-
-| # | Priority | Action |
-|---|----------|--------|
-| 1 | Critical | Fix `clusters_labels` typo → `cluster_labels` |
-| 2 | Critical | Replace bare `except:` with specific exception types |
-| 3 | Critical | Add thread-safe locking for concurrent parsing |
-| 4 | High | Add division-by-zero guard in Cramér's V |
-| 5 | High | Restrict CORS to frontend origin |
-| 6 | High | Add unit tests for core pipeline |
-| 7 | Medium | Split `UAPAnalyzer` into focused services |
-| 8 | Medium | Implement temporal and geospatial analysis features |
-| 9 | Medium | Add proper logging and performance instrumentation |
-| 10 | Low | Clean up unused dependencies |

codex_app_analysis.md

@@ -1,148 +0,0 @@
-# UAP Analysis App: Improvement and Debugging Notes
-
-## Scope Reviewed
-- Backend API: `backend/main.py`
-- Frontend app/store/pages: `frontend/src/**`
-- Project positioning/features: `README.md`
-
-## High-Priority Debugging and Reliability Issues
-
-### 1) Global in-memory backend state is shared across all users/sessions
-- File: `backend/main.py`
-- Current behavior: `state` is a single process-level dictionary storing dataset, filtered data, analysis results, and query context.
-- Risk:
-  - Cross-user data leakage.
-  - Race conditions (one user can overwrite another user’s data mid-session).
-  - Non-deterministic behavior in multi-worker deployment.
-- Improvement:
-  - Introduce per-session/project IDs and isolate state in Redis or a DB cache keyed by session.
-  - Add dataset ownership + TTL cleanup.
-
-### 2) Dashboard “Analysis Runs” counter is effectively broken
-- File: `backend/main.py`
-- Current behavior: `/api/dashboard/summary` reports `analyzed_columns` from `state["col_names"]`, but `col_names` is never populated in `run_analysis`.
-- Impact: dashboard can show incorrect/always-zero run stats.
-- Improvement:
-  - Set `state["col_names"] = req.columns` (validated columns only) during analysis.
-  - Consider storing analysis timestamp and run count.
-
-### 3) Numeric filtering ignores partial ranges
-- Files: `backend/main.py`, `frontend/src/components/data/FilterPanel.tsx`
-- Current behavior: numeric filter only applies when both `min_val` and `max_val` are provided.
-- Impact: user-provided minimum-only or maximum-only constraints do nothing.
-- Improvement:
-  - Support all combinations: min-only, max-only, and bounded range.
-
-### 4) Filtering error handling is silent on frontend
-- File: `frontend/src/components/data/DataExplorer.tsx`
-- Current behavior: `handleFilter` catches errors and suppresses them.
-- Impact: user cannot tell whether filtering failed or returned no matches.
-- Improvement:
-  - Surface backend error in UI (same pattern used by upload/load errors).
-
-### 5) Backend query endpoint has expensive prompt construction and token-risk
-- File: `backend/main.py`
-- Current behavior: concatenates up to 500 rows of full text into one prompt string.
-- Risks:
-  - Large latency/cost spikes.
-  - Prompt truncation or model failure on long text columns.
-- Improvement:
-  - Add token-aware chunking/sampling and optional map-reduce summarization.
-  - Enforce max characters/tokens per request with clear user feedback.
-
-### 6) CORS config is overly permissive and potentially invalid for credentials
-- File: `backend/main.py`
-- Current behavior: `allow_origins=["*"]` with `allow_credentials=True`.
-- Risk: browser credential behavior can be inconsistent; security posture is weak for production.
-- Improvement:
-  - Use explicit origin allowlist per environment.
-  - Keep credentials disabled unless required.
-
-## Data-Analysis Quality Gaps (Core Product)
-
-### 7) Analysis pipeline is currently mock/simulated, not aligned with README claims
-- File: `backend/main.py`
-- Current behavior: analysis uses value counts + random 2D points + correlation proxy for “XGBoost-like” output.
-- Impact: users may interpret synthetic outputs as real model outputs.
-- Improvement:
-  - Label this mode explicitly as `demo/mock` in API/UI.
-  - Add a production pipeline path using real embeddings + UMAP/HDBSCAN + trained model artifacts.
-
-### 8) Cluster assignment is too coarse for high-cardinality text fields
-- File: `backend/main.py`
-- Current behavior: top 32 frequent values become “clusters”; all others mapped to `Other`.
-- Impact: weak signal extraction for long-tail UAP narratives.
-- Improvement:
-  - Use embedding-based similarity clustering with min-cluster-size tuning.
-  - Keep top terms as labels only, not as cluster definitions.
-
-### 9) “XGBoost” results are correlation-based placeholders
-- File: `backend/main.py`
-- Current behavior: feature importance derived from absolute correlation among category codes, with random “accuracy”.
-- Impact: misleading ML interpretation.
-- Improvement:
-  - Either rename section (`Association Importance`) or run real train/validation with metrics and confidence intervals.
-
-### 10) Cramer’s V stability safeguards are minimal
-- File: `backend/main.py`
-- Current behavior: exceptions are swallowed to `0.0` values.
-- Impact: matrix can hide data-quality problems.
-- Improvement:
-  - Return diagnostics (insufficient contingency size, sparse table warning, low sample counts).
-
-## UX and Feature Improvements for Analysis Workflows
-
-### 11) Add reproducibility controls
-- Current gap: random projections are generated without surfaced seed controls; pipeline details are hidden.
-- Improvement:
-  - UI inputs for random seed and analysis config profile.
-  - Persist configuration alongside results.
-
-### 12) Add time/location-first analysis modules
-- Context: UAP datasets are usually spatiotemporal.
-- Improvement ideas:
-  - Temporal anomaly detection (daily/weekly trend breaks).
-  - Geo heatmaps + hotspot evolution over time.
-  - Co-occurrence matrices for shape/light/motion features.
-
-### 13) Add model/result provenance panel
-- Improvement:
-  - Track dataset hash, row count after filters, analysis timestamp, selected columns, pipeline version.
-  - Show this metadata in Analysis and export payloads.
-
-### 14) Improve filter capabilities for real EDA
-- Current gap: categorical filter relies on top-values list and cannot easily search rare categories.
-- Improvement:
-  - Add searchable categorical picker and “include nulls/exclude nulls”.
-  - Add reusable saved filter presets.
-
-### 15) Add export/reporting features
-- Improvement:
-  - Export filtered dataset, correlation matrix, and feature-importance JSON/CSV.
-  - One-click markdown/PDF report with charts and configuration metadata.
-
-## Engineering Quality and Maintainability
-
-### 16) Add automated tests for core API behaviors
-- Suggested minimal suite:
-  - `/api/data/load`, `/api/data/filter`, `/api/analyze/run`, `/api/dashboard/summary`, `/api/query/gemini` failure paths.
-  - Numeric filter edge cases (min-only/max-only).
-  - State isolation once sessionized.
-
-### 17) Add request/analysis observability
-- Improvement:
-  - Structured logging + request IDs.
-  - Timing metrics per stage (load/filter/analyze/query).
-  - Distinguish user errors (4xx) from pipeline errors (5xx).
-
-### 18) Clarify mode separation: demo vs production
-- Improvement:
-  - Feature flags/environment variable to select mock vs full analysis backend.
-  - UI badges and warning copy to prevent scientific misinterpretation.
-
-## Suggested Implementation Order
-1. Fix state isolation and dashboard run-count correctness.
-2. Fix filtering behavior + frontend error surfacing.
-3. Mark current analysis mode as mock and rename misleading outputs.
-4. Add reproducibility/provenance metadata and exports.
-5. Introduce real embedding + clustering + model pipeline behind a feature flag.

embed_csv.py

@@ -1,91 +0,0 @@
-"""
-embed_csv.py — batch-embed a CSV column and save to HDF5
-
-Usage:
-    uv run python embed_csv.py --input data.csv --column description
-    uv run python embed_csv.py --input data.csv --column description --output out.h5 --batch-size 128 --prompt web_search_query
-"""
-
-import argparse
-import sys
-import time
-
-import numpy as np
-import pandas as pd
-import torch
-from sentence_transformers import SentenceTransformer
-from tqdm import tqdm
-
-MODEL_ID = "microsoft/harrier-oss-v1-0.6b"
-
-
-def load_model(device: str) -> SentenceTransformer:
-    print(f"Loading {MODEL_ID} on {device}…")
-    t0 = time.time()
-    model_kwargs = {"dtype": "auto"}
-    if device == "cuda":
-        model_kwargs["device_map"] = "cuda"  # load directly into VRAM, skip CPU copy
-    model = SentenceTransformer(MODEL_ID, model_kwargs=model_kwargs)
-    if device != "cuda":
-        model.to(device)
-    print(f"Model ready in {time.time() - t0:.1f}s")
-    return model
-
-
-def embed(model: SentenceTransformer, texts: list[str], batch_size: int,
-          prompt_name: str | None) -> np.ndarray:
-    batches = [texts[i:i + batch_size] for i in range(0, len(texts), batch_size)]
-    kwargs = {"prompt_name": prompt_name} if prompt_name else {}
-    all_embs = []
-    for batch in tqdm(batches, desc="Encoding", unit="batch"):
-        with torch.no_grad():
-            all_embs.append(model.encode(batch, show_progress_bar=False, **kwargs))
-    return np.vstack(all_embs)
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Embed a CSV column with harrier-oss-v1-0.6b")
-    parser.add_argument("--input",      required=True,  help="Input CSV file")
-    parser.add_argument("--column",     required=True,  help="Column to embed")
-    parser.add_argument("--output",     default=None,   help="Output .h5 file (default: <input>_embeddings.h5)")
-    parser.add_argument("--key",        default="df",   help="HDF5 key (default: df)")
-    parser.add_argument("--batch-size", default=256,    type=int, help="Batch size (default: 256)")
-    parser.add_argument("--prompt",     default=None,   choices=["web_search_query"],
-                        help="Prompt name for query encoding (omit for documents)")
-    args = parser.parse_args()
-
-    # Output path
-    out_path = args.output or args.input.rsplit(".", 1)[0] + "_embeddings.h5"
-
-    # Load CSV
-    print(f"Reading {args.input}…")
-    df = pd.read_csv(args.input)
-    if args.column not in df.columns:
-        print(f"ERROR: column '{args.column}' not found. Available: {list(df.columns)}")
-        sys.exit(1)
-    print(f"{len(df):,} rows loaded. Embedding column: '{args.column}'")
-
-    texts = df[args.column].fillna("").astype(str).tolist()
-
-    # Device
-    device = "cuda" if torch.cuda.is_available() else "cpu"
-    if device == "cuda":
-        print(f"GPU: {torch.cuda.get_device_name(0)}")
-
-    model = load_model(device)
-
-    # Embed
-    t0 = time.time()
-    embeddings = embed(model, texts, batch_size=args.batch_size, prompt_name=args.prompt)
-    elapsed = time.time() - t0
-    print(f"Done in {elapsed:.1f}s — shape {embeddings.shape} "
-          f"({len(texts)/elapsed:.0f} texts/s)")
-
-    # Save
-    df["embeddings"] = embeddings.tolist()
-    df.to_hdf(out_path, key=args.key, mode="w")
-    print(f"Saved → {out_path}  (key='{args.key}')")
-
-
-if __name__ == "__main__":
-    main()

embeddings.py

@@ -1,259 +0,0 @@
-"""
-Self-contained multimodal embedding helper — Python port of utils/vertex-embeddings.ts.
-
-Generates embeddings via Google Gemini (gemini-embedding-2-preview, 768-dim).
-Mirrors the image / text / multimodal entry points used by app/api/embeddings/.
-Storage / pgvector search / clustering are intentionally NOT included — this
-module is pure embedding generation. Drop into any Python project.
-
-Setup:
-    pip install google-genai pillow requests
-    export GEMINI_API_KEY=...
-
-Usage:
-    from embeddings import (
-        generate_image_embedding,
-        generate_text_embedding,
-        generate_multimodal_embedding,
-        cosine_similarity,
-    )
-
-    vec_img = generate_image_embedding("https://example.com/chair.jpg")
-    vec_txt = generate_text_embedding("modern walnut dining chair")
-    vec_mm  = generate_multimodal_embedding(
-        "https://example.com/chair.jpg", "modern walnut dining chair"
-    )
-    print(cosine_similarity(vec_img, vec_mm))
-
-CLI:
-    python embeddings.py text "modern walnut dining chair"
-    python embeddings.py image https://example.com/chair.jpg
-    python embeddings.py image ./local/chair.jpg
-    python embeddings.py multimodal ./local/chair.jpg "modern walnut dining chair"
-"""
-
-from __future__ import annotations
-
-import argparse
-import base64
-import io
-import json
-import math
-import os
-import sys
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Literal, Optional
-
-import requests
-from google import genai
-from google.genai import types as genai_types
-from PIL import Image
-
-EMBEDDING_MODEL = "gemini-embedding-2-preview"
-EMBEDDING_DIMENSIONS = 768
-
-# Pillow can decode these but Gemini may reject them — normalise to JPEG.
-UNSUPPORTED_MIME = {"image/webp", "image/tiff", "image/bmp", "image/avif"}
-
-TaskType = Literal["RETRIEVAL_DOCUMENT", "RETRIEVAL_QUERY"]
-
-_client: Optional[genai.Client] = None
-
-
-def _get_client() -> genai.Client:
-    """Lazily-cached Gemini client. Reads GEMINI_API_KEY from environment."""
-    global _client
-    if _client is not None:
-        return _client
-    api_key = os.environ.get("GEMINI_API_KEY")
-    if not api_key:
-        raise RuntimeError("GEMINI_API_KEY environment variable is not set")
-    _client = genai.Client(api_key=api_key)
-    return _client
-
-
-@dataclass
-class _ImageBytes:
-    data: bytes
-    mime_type: str
-
-
-def _ensure_jpeg(raw: bytes, mime: str) -> _ImageBytes:
-    """Convert webp/tiff/bmp/avif → jpeg. Mirrors ensureJpeg() in the TS module."""
-    if mime.lower() not in UNSUPPORTED_MIME:
-        return _ImageBytes(raw, mime)
-    img = Image.open(io.BytesIO(raw))
-    if img.mode in ("RGBA", "LA", "P"):
-        img = img.convert("RGB")
-    buf = io.BytesIO()
-    img.save(buf, format="JPEG", quality=90)
-    return _ImageBytes(buf.getvalue(), "image/jpeg")
-
-
-def _load_image(source: str) -> _ImageBytes:
-    """Load image bytes from a URL, local path, or `data:` URL."""
-    if source.startswith("data:"):
-        return _parse_data_url(source)
-    if source.startswith(("http://", "https://")):
-        resp = requests.get(source, timeout=30)
-        resp.raise_for_status()
-        mime = resp.headers.get("content-type", "image/jpeg").split(";", 1)[0]
-        return _ensure_jpeg(resp.content, mime)
-    p = Path(source).expanduser()
-    if not p.is_file():
-        raise FileNotFoundError(f"Image not found: {source}")
-    suffix = p.suffix.lower().lstrip(".")
-    mime = {
-        "jpg": "image/jpeg",
-        "jpeg": "image/jpeg",
-        "png": "image/png",
-        "gif": "image/gif",
-        "webp": "image/webp",
-        "tiff": "image/tiff",
-        "tif": "image/tiff",
-        "bmp": "image/bmp",
-        "avif": "image/avif",
-    }.get(suffix, "image/jpeg")
-    return _ensure_jpeg(p.read_bytes(), mime)
-
-
-def _parse_data_url(data_url: str) -> _ImageBytes:
-    """Parse `data:image/...;base64,...` into raw bytes + mime."""
-    if not data_url.startswith("data:"):
-        raise ValueError("Not a data URL")
-    header, _, b64 = data_url.partition(",")
-    if not b64:
-        raise ValueError("Malformed data URL")
-    mime = header[5 : header.index(";")] if ";" in header else header[5:]
-    return _ensure_jpeg(base64.b64decode(b64), mime)
-
-
-def _embed(parts: list, task_type: TaskType) -> list[float]:
-    """Single-call wrapper around client.models.embed_content."""
-    client = _get_client()
-    if len(parts) == 1 and isinstance(parts[0], str):
-        contents = parts[0]
-    else:
-        contents = genai_types.Content(parts=parts)
-    result = client.models.embed_content(
-        model=EMBEDDING_MODEL,
-        contents=contents,
-        config=genai_types.EmbedContentConfig(
-            output_dimensionality=EMBEDDING_DIMENSIONS,
-            task_type=task_type,
-        ),
-    )
-    if not result.embeddings or not result.embeddings[0].values:
-        raise RuntimeError("No embedding returned from model")
-    return list(result.embeddings[0].values)
-
-
-def generate_image_embedding(
-    source: str, *, task_type: TaskType = "RETRIEVAL_DOCUMENT"
-) -> list[float]:
-    """Embed an image given its URL, local path, or `data:` URL."""
-    img = _load_image(source)
-    part = genai_types.Part(
-        inline_data=genai_types.Blob(mime_type=img.mime_type, data=img.data)
-    )
-    return _embed([part], task_type)
-
-
-def generate_text_embedding(
-    text: str, *, task_type: TaskType = "RETRIEVAL_QUERY"
-) -> list[float]:
-    """Embed a piece of text. Default task_type matches the TS search path."""
-    if not text:
-        raise ValueError("text must be non-empty")
-    return _embed([text], task_type)
-
-
-def generate_multimodal_embedding(
-    image_source: str,
-    text: str,
-    *,
-    task_type: TaskType = "RETRIEVAL_DOCUMENT",
-) -> list[float]:
-    """Embed an image + text pair as a single 768-dim vector."""
-    if not text:
-        raise ValueError("text must be non-empty for multimodal embedding")
-    img = _load_image(image_source)
-    parts = [
-        genai_types.Part(
-            inline_data=genai_types.Blob(mime_type=img.mime_type, data=img.data)
-        ),
-        genai_types.Part(text=text),
-    ]
-    return _embed(parts, task_type)
-
-
-def cosine_similarity(a: list[float], b: list[float]) -> float:
-    """Cosine similarity in [-1, 1]. Returns 0.0 if either vector is zero."""
-    if len(a) != len(b):
-        raise ValueError(f"vector length mismatch: {len(a)} vs {len(b)}")
-    dot = mag_a = mag_b = 0.0
-    for x, y in zip(a, b):
-        dot += x * y
-        mag_a += x * x
-        mag_b += y * y
-    if mag_a == 0.0 or mag_b == 0.0:
-        return 0.0
-    return dot / (math.sqrt(mag_a) * math.sqrt(mag_b))
-
-
-def _cli() -> int:
-    parser = argparse.ArgumentParser(
-        description="Generate Gemini multimodal embeddings."
-    )
-    sub = parser.add_subparsers(dest="cmd", required=True)
-
-    p_text = sub.add_parser("text", help="Embed a text string.")
-    p_text.add_argument("text")
-
-    p_img = sub.add_parser("image", help="Embed an image (URL, path, or data URL).")
-    p_img.add_argument("source")
-    p_img.add_argument(
-        "--query",
-        action="store_true",
-        help="Use RETRIEVAL_QUERY task type (default is RETRIEVAL_DOCUMENT).",
-    )
-
-    p_mm = sub.add_parser("multimodal", help="Embed image + text together.")
-    p_mm.add_argument("source")
-    p_mm.add_argument("text")
-
-    p_sim = sub.add_parser(
-        "similarity",
-        help="Compute cosine similarity between two embeddings (read JSON arrays from stdin or file).",
-    )
-    p_sim.add_argument("a")
-    p_sim.add_argument("b")
-
-    args = parser.parse_args()
-
-    def _print_vec(vec: list[float]) -> None:
-        print(json.dumps(vec))
-
-    if args.cmd == "text":
-        _print_vec(generate_text_embedding(args.text))
-        return 0
-    if args.cmd == "image":
-        task: TaskType = "RETRIEVAL_QUERY" if args.query else "RETRIEVAL_DOCUMENT"
-        _print_vec(generate_image_embedding(args.source, task_type=task))
-        return 0
-    if args.cmd == "multimodal":
-        _print_vec(generate_multimodal_embedding(args.source, args.text))
-        return 0
-    if args.cmd == "similarity":
-        a = json.loads(Path(args.a).read_text() if Path(args.a).is_file() else args.a)
-        b = json.loads(Path(args.b).read_text() if Path(args.b).is_file() else args.b)
-        print(cosine_similarity(a, b))
-        return 0
-
-    parser.print_help()
-    return 2
-
-
-if __name__ == "__main__":
-    sys.exit(_cli())

final_ufoseti_dataset.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:829fb6660b24626eb5db39952783c6e17dc17c7c4636df0dfc8b641d0c84efe5
+size 39219544

frontend/.gitignore

@@ -1,24 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-node_modules
-dist
-dist-ssr
-*.local
-
-# Editor directories and files
-.vscode/*
-!.vscode/extensions.json
-.idea
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?

frontend/DEPLOY_VERCEL.md

@@ -1,74 +0,0 @@
-# Deploying the frontend to Vercel
-
-The React/Vite frontend deploys cleanly to Vercel. The FastAPI backend (`api/`)
-does **not** — it needs a persistent, GPU-capable container (torch,
-sentence-transformers, in-memory session state, 240 s analysis jobs), which
-Vercel's serverless functions can't host. So this is a **split deploy**:
-
-```
-  Browser ──▶ Vercel (static React SPA)
-                 │  /api/* calls
-                 ▼
-            Backend host (Hugging Face Spaces / Render / Railway)
-            FastAPI + ML stack, reads the .h5 datasets
-```
-
-## 1. Deploy the backend first
-
-The app is already configured for **Hugging Face Spaces** (see `README.md`
-metadata) — that's the recommended host because it runs the full ML stack and
-offers GPU tiers for the real embedding pipeline (`UAP_ANALYSIS_MODE=production`).
-Render or Railway also work for CPU/mock mode.
-
-Whichever you pick, set these env vars on the backend:
-
-| Var | Value |
-|-----|-------|
-| `UAP_API_CORS_ORIGINS` | Your Vercel URL, e.g. `https://uap.vercel.app` (comma-separated for multiple) |
-| `UAP_ANALYSIS_MODE` | `production` (real clustering) or `mock` (fast, no GPU) |
-
-Note the backend's public URL — e.g. `https://<user>-uap.hf.space`.
-
-## 2. Deploy the frontend to Vercel
-
-1. **New Project** → import this repo.
-2. Set **Root Directory** to `frontend` (the repo root is a Python project).
-   Vercel auto-detects Vite from there; `vercel.json` pins the build.
-3. Add an **Environment Variable**:
-
-   | Name | Value |
-   |------|-------|
-   | `VITE_API_BASE` | The backend origin, e.g. `https://<user>-uap.hf.space` |
-
-   `VITE_*` vars are inlined at build time, so changing it later needs a redeploy.
-4. **Deploy.**
-
-The client reads `VITE_API_BASE` (`src/api/client.ts`) and calls
-`<VITE_API_BASE>/api/...`. When the var is unset it falls back to a same-origin
-`/api`, so **local dev is unchanged** — `npm run dev` still proxies `/api` to
-`localhost:8000` via `vite.config.ts`.
-
-### Alternative: same-origin proxy (no CORS)
-
-Instead of `VITE_API_BASE` + backend CORS, you can leave the env var unset and
-proxy `/api` through Vercel. Add this to `vercel.json` **above** the SPA
-fallback rewrite and you won't touch the backend's CORS at all:
-
-```json
-{ "source": "/api/:path*", "destination": "https://<user>-uap.hf.space/api/:path*" }
-```
-
-## CLI deploy (optional)
-
-```bash
-cd frontend
-npx vercel            # preview deploy, prompts for login + project link
-npx vercel --prod     # production deploy
-```
-
-## What does NOT go to Vercel
-
-- `api/` (FastAPI) — backend host only.
-- The `.h5` datasets and the 124 MB `uap_clusters_llm.html` — these stay with
-  the backend, which serves the cluster viz via `/api/analysis/clusters`. They
-  are not bundled into the 5 MB Vercel build.

frontend/README.md

@@ -1,73 +0,0 @@
-# React + TypeScript + Vite
-
-This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
-
-Currently, two official plugins are available:
-
-- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) (or [oxc](https://oxc.rs) when used in [rolldown-vite](https://vite.dev/guide/rolldown)) for Fast Refresh
-- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
-
-## React Compiler
-
-The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation).
-
-## Expanding the ESLint configuration
-
-If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules:
-
-```js
-export default defineConfig([
-  globalIgnores(['dist']),
-  {
-    files: ['**/*.{ts,tsx}'],
-    extends: [
-      // Other configs...
-
-      // Remove tseslint.configs.recommended and replace with this
-      tseslint.configs.recommendedTypeChecked,
-      // Alternatively, use this for stricter rules
-      tseslint.configs.strictTypeChecked,
-      // Optionally, add this for stylistic rules
-      tseslint.configs.stylisticTypeChecked,
-
-      // Other configs...
-    ],
-    languageOptions: {
-      parserOptions: {
-        project: ['./tsconfig.node.json', './tsconfig.app.json'],
-        tsconfigRootDir: import.meta.dirname,
-      },
-      // other options...
-    },
-  },
-])
-```
-
-You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules:
-
-```js
-// eslint.config.js
-import reactX from 'eslint-plugin-react-x'
-import reactDom from 'eslint-plugin-react-dom'
-
-export default defineConfig([
-  globalIgnores(['dist']),
-  {
-    files: ['**/*.{ts,tsx}'],
-    extends: [
-      // Other configs...
-      // Enable lint rules for React
-      reactX.configs['recommended-typescript'],
-      // Enable lint rules for React DOM
-      reactDom.configs.recommended,
-    ],
-    languageOptions: {
-      parserOptions: {
-        project: ['./tsconfig.node.json', './tsconfig.app.json'],
-        tsconfigRootDir: import.meta.dirname,
-      },
-      // other options...
-    },
-  },
-])
-```

frontend/eslint.config.js

@@ -1,23 +0,0 @@
-import js from '@eslint/js'
-import globals from 'globals'
-import reactHooks from 'eslint-plugin-react-hooks'
-import reactRefresh from 'eslint-plugin-react-refresh'
-import tseslint from 'typescript-eslint'
-import { defineConfig, globalIgnores } from 'eslint/config'
-
-export default defineConfig([
-  globalIgnores(['dist']),
-  {
-    files: ['**/*.{ts,tsx}'],
-    extends: [
-      js.configs.recommended,
-      tseslint.configs.recommended,
-      reactHooks.configs.flat.recommended,
-      reactRefresh.configs.vite,
-    ],
-    languageOptions: {
-      ecmaVersion: 2020,
-      globals: globals.browser,
-    },
-  },
-])

frontend/index.html

@@ -1,16 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><text y='.9em' font-size='90'>&#x2B22;</text></svg>" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>UAP Foundry | Analysis Platform</title>
-    <link rel="preconnect" href="https://fonts.googleapis.com" />
-    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
-    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap" rel="stylesheet" />
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>

frontend/package.json

@@ -1,39 +0,0 @@
-{
-  "name": "frontend",
-  "private": true,
-  "version": "0.0.0",
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "tsc -b && vite build",
-    "lint": "eslint .",
-    "preview": "vite preview"
-  },
-  "dependencies": {
-    "@tailwindcss/vite": "^4.1.18",
-    "@types/react-plotly.js": "^2.6.4",
-    "lucide-react": "^0.564.0",
-    "plotly.js": "^3.3.1",
-    "react": "^19.2.0",
-    "react-dom": "^19.2.0",
-    "react-plotly.js": "^2.6.0",
-    "react-router-dom": "^7.13.0",
-    "statsmodels": "^0.0.1-security",
-    "tailwindcss": "^4.1.18",
-    "zustand": "^5.0.11"
-  },
-  "devDependencies": {
-    "@eslint/js": "^9.39.1",
-    "@types/node": "^24.10.1",
-    "@types/react": "^19.2.7",
-    "@types/react-dom": "^19.2.3",
-    "@vitejs/plugin-react": "^5.1.1",
-    "eslint": "^9.39.1",
-    "eslint-plugin-react-hooks": "^7.0.1",
-    "eslint-plugin-react-refresh": "^0.4.24",
-    "globals": "^16.5.0",
-    "typescript": "~5.9.3",
-    "typescript-eslint": "^8.48.0",
-    "vite": "^7.3.1"
-  }
-}

frontend/src/App.tsx

@@ -1,33 +0,0 @@
-import { AppShell } from './components/layout/AppShell';
-import { Dashboard } from './components/dashboard/Dashboard';
-import { DataExplorer } from './components/data/DataExplorer';
-import { ParsingPage } from './components/parsing/ParsingPage';
-import { AnalysisPage } from './components/analysis/AnalysisPage';
-import { QueryPage } from './components/query/QueryPage';
-import { RagSearchPage } from './components/rag/RagSearchPage';
-import { ScuPage } from './components/scu/ScuPage';
-import { MapPage } from './components/map/MapPage';
-import { MagneticPage } from './components/magnetic/MagneticPage';
-import { ClusterView } from './components/analysis/ClusterView';
-import { useStore } from './store/useStore';
-
-function App() {
-  const { currentPage } = useStore();
-
-  return (
-    <AppShell>
-      {currentPage === 'dashboard' && <Dashboard />}
-      {currentPage === 'data' && <DataExplorer />}
-      {currentPage === 'parsing' && <ParsingPage />}
-      {currentPage === 'analysis' && <AnalysisPage />}
-      {currentPage === 'query' && <QueryPage />}
-      {currentPage === 'rag' && <RagSearchPage />}
-      {currentPage === 'scu' && <ScuPage />}
-      {currentPage === 'map' && <MapPage />}
-      {currentPage === 'magnetic' && <MagneticPage />}
-      {currentPage === 'clusters' && <ClusterView />}
-    </AppShell>
-  );
-}
-
-export default App;

frontend/src/api/client.ts

@@ -1,264 +0,0 @@
-import type {
-  LoadDataResponse,
-  AnalysisResponse,
-  DashboardSummary,
-  SchemaListResponse,
-  SchemaMergeResponse,
-  SchemaCoverageResponse,
-  ParseUploadResponse,
-  CostEstimate,
-  ParseRunResponse,
-  ScuCriteriaResponse,
-  ScuNormalizeResponse,
-  ScuFilterResponse,
-  RagSearchResponse,
-  CramersVResponse,
-  ContingencyResponse,
-  ColumnGroupsResponse,
-  XgboostImportanceResponse,
-} from '../types';
-
-// API origin is configurable for split deployments (e.g. frontend on Vercel,
-// backend on Hugging Face Spaces / Render). Set VITE_API_BASE to the backend
-// origin at build time, e.g. "https://user-uap.hf.space". When unset it falls
-// back to a same-origin "/api", which the Vite dev proxy (vite.config.ts) and
-// a Vercel `/api` rewrite both handle transparently.
-const API_ORIGIN = (import.meta.env.VITE_API_BASE ?? '').replace(/\/+$/, '');
-const BASE = `${API_ORIGIN}/api`;
-
-async function request<T>(url: string, init?: RequestInit): Promise<T> {
-  const res = await fetch(`${BASE}${url}`, {
-    headers: { 'Content-Type': 'application/json' },
-    ...init,
-  });
-  if (!res.ok) {
-    const body = await res.json().catch(() => ({ detail: res.statusText }));
-    throw new Error(body.detail || `Request failed: ${res.status}`);
-  }
-  return res.json();
-}
-
-export const api = {
-  loadData(type = 'west', rows = 15000): Promise<LoadDataResponse> {
-    return request(`/data/load?type=${type}&rows=${rows}`);
-  },
-
-  uploadFile(file: File): Promise<LoadDataResponse> {
-    const form = new FormData();
-    form.append('file', file);
-    return fetch(`${BASE}/data/upload`, { method: 'POST', body: form }).then(
-      async (res) => {
-        if (!res.ok) {
-          const body = await res.json().catch(() => ({ detail: res.statusText }));
-          throw new Error(body.detail || `Upload failed: ${res.status}`);
-        }
-        return res.json();
-      }
-    );
-  },
-
-  filterData(
-    filters: {
-      column: string;
-      type: string;
-      values?: string[];
-      min_val?: number;
-      max_val?: number;
-      pattern?: string;
-    }[]
-  ): Promise<LoadDataResponse> {
-    return request('/data/filter', {
-      method: 'POST',
-      body: JSON.stringify(filters),
-    });
-  },
-
-  getColumns(): Promise<{ columns: { name: string; dtype: string; unique: number; non_null: number }[] }> {
-    return request('/data/columns');
-  },
-
-  getColumnValues(
-    column: string,
-    search = '',
-    limit = 50
-  ): Promise<{
-    column: string;
-    values: { value: string; count: number }[];
-    total_matches: number;
-  }> {
-    return request(
-      `/data/column-values?column=${encodeURIComponent(column)}&search=${encodeURIComponent(search)}&limit=${limit}`
-    );
-  },
-
-  runAnalysis(
-    columns: string[],
-    opts: {
-      enable_tfidf?: boolean;
-      min_cluster_size?: number;
-      n_neighbors?: number;
-      min_dist?: number;
-      top_n?: number;
-    } = {}
-  ): Promise<AnalysisResponse> {
-    return request('/analyze/run', {
-      method: 'POST',
-      body: JSON.stringify({ columns, ...opts }),
-    });
-  },
-
-  // ── Parsing ───────────────────────────────────────────────────────────
-  getSchemas(): Promise<SchemaListResponse> {
-    return request('/parse/schemas');
-  },
-
-  mergeSchema(
-    labels: string[],
-    customFields?: Record<string, unknown>
-  ): Promise<SchemaMergeResponse> {
-    return request('/parse/schema-merge', {
-      method: 'POST',
-      body: JSON.stringify({ labels, custom_fields: customFields ?? null }),
-    });
-  },
-
-  schemaCoverage(
-    labels: string[],
-    columns: string[],
-    customFields?: Record<string, unknown>
-  ): Promise<SchemaCoverageResponse> {
-    return request('/parse/schema-coverage', {
-      method: 'POST',
-      body: JSON.stringify({ labels, columns, custom_fields: customFields ?? null }),
-    });
-  },
-
-  uploadParseFile(file: File): Promise<ParseUploadResponse> {
-    const form = new FormData();
-    form.append('file', file);
-    return fetch(`${BASE}/parse/upload`, { method: 'POST', body: form }).then(
-      async (res) => {
-        if (!res.ok) {
-          const body = await res.json().catch(() => ({ detail: res.statusText }));
-          throw new Error(body.detail || `Upload failed: ${res.status}`);
-        }
-        return res.json();
-      }
-    );
-  },
-
-  estimateParse(
-    columns: string[],
-    formatJson: string,
-    model: string,
-    useBatch = false
-  ): Promise<CostEstimate> {
-    return request('/parse/estimate', {
-      method: 'POST',
-      body: JSON.stringify({ columns, format_json: formatJson, model, use_batch: useBatch }),
-    });
-  },
-
-  runParse(payload: {
-    columns: string[];
-    format_json: string;
-    provider: string;
-    model: string;
-    api_key: string;
-    max_workers?: number;
-    keep_columns?: string[];
-  }): Promise<ParseRunResponse> {
-    return request('/parse/run', {
-      method: 'POST',
-      body: JSON.stringify(payload),
-    });
-  },
-
-  // ── SCU normalization ─────────────────────────────────────────────────
-  getScuCriteria(): Promise<ScuCriteriaResponse> {
-    return request('/scu/criteria');
-  },
-
-  scuNormalize(): Promise<ScuNormalizeResponse> {
-    return request('/scu/normalize', { method: 'POST', body: '{}' });
-  },
-
-  scuFilter(criterionKeys: string[]): Promise<ScuFilterResponse> {
-    return request('/scu/filter', {
-      method: 'POST',
-      body: JSON.stringify({ criterion_keys: criterionKeys }),
-    });
-  },
-
-  // ── RAG search (Cohere) ───────────────────────────────────────────────
-  ragSearch(
-    columns: string[],
-    question: string,
-    cohereKey: string,
-    topN = 50
-  ): Promise<RagSearchResponse> {
-    return request('/rag/search', {
-      method: 'POST',
-      body: JSON.stringify({ columns, question, cohere_key: cohereKey, top_n: topN }),
-    });
-  },
-
-  // ── Cramér's V explorer ───────────────────────────────────────────────
-  cramersV(payload: {
-    columns?: string[] | null;
-    drop_missing?: boolean;
-    exclude_trivial?: boolean;
-    strong_threshold?: number;
-    high_threshold?: number;
-    source?: string;
-  }): Promise<CramersVResponse> {
-    return request('/analysis/cramers-v', {
-      method: 'POST',
-      body: JSON.stringify(payload),
-    });
-  },
-
-  contingency(payload: {
-    col1: string;
-    col2: string;
-    drop_missing?: boolean;
-    source?: string;
-  }): Promise<ContingencyResponse> {
-    return request('/analysis/contingency', {
-      method: 'POST',
-      body: JSON.stringify(payload),
-    });
-  },
-
-  columnGroups(payload: {
-    source?: string;
-    high_threshold?: number;
-  }): Promise<ColumnGroupsResponse> {
-    return request('/analysis/column-groups', {
-      method: 'POST',
-      body: JSON.stringify(payload),
-    });
-  },
-
-  xgboostImportance(columns: string[], source = 'dataset'): Promise<XgboostImportanceResponse> {
-    return request('/analysis/xgboost', {
-      method: 'POST',
-      body: JSON.stringify({ columns, source }),
-    });
-  },
-
-  queryGemini(question: string, columns: string[], geminiKey: string): Promise<{ status: string; response: string; context_rows_used?: number; columns_used?: string[] }> {
-    return request('/query/gemini', {
-      method: 'POST',
-      body: JSON.stringify({ question, columns, gemini_key: geminiKey }),
-    });
-  },
-
-  getDashboardSummary(): Promise<DashboardSummary> {
-    return request('/dashboard/summary');
-  },
-
-  healthCheck(): Promise<{ status: string; version: string }> {
-    return request('/health');
-  },
-};

frontend/src/components/analysis/AnalysisPage.tsx

@@ -1,322 +0,0 @@
-import { useState } from 'react';
-import { Play, AlertTriangle, CheckCircle2, Layers, BarChart3, Grid3x3, Settings2, Network } from 'lucide-react';
-import { api } from '../../api/client';
-import { useStore } from '../../store/useStore';
-import { Panel } from '../common/Panel';
-import { LoadingSpinner } from '../common/LoadingSpinner';
-import { ClusterVisualization } from './ClusterVisualization';
-import { CorrelationHeatmap } from './CorrelationHeatmap';
-import { XGBoostResults } from './XGBoostResults';
-import { DistributionChart } from './DistributionChart';
-import { CramersVExplorer } from './CramersVExplorer';
-import type { AnalysisResponse, XGBoostResult } from '../../types';
-
-type TabId = 'clusters' | 'correlation' | 'xgboost' | 'distribution' | 'association';
-
-export function AnalysisPage() {
-  const { data, dataLoaded, analysisResults, setAnalysisResults, analysisRunning, setAnalysisRunning, setPage } = useStore();
-  const [selected, setSelected] = useState<string[]>([]);
-  const [error, setError] = useState<string | null>(null);
-  const [activeTab, setActiveTab] = useState<TabId>('clusters');
-  // XGBoost feature importance handed over from the Cramér's V explorer.
-  const [assocXgboost, setAssocXgboost] = useState<Record<string, XGBoostResult> | null>(null);
-
-  const handleAssocXgboost = (r: Record<string, XGBoostResult>) => {
-    setAssocXgboost(r);
-    setActiveTab('xgboost');
-  };
-
-  // Cluster pipeline tuning (mirrors analyzing.py controls)
-  const [showParams, setShowParams] = useState(false);
-  const [enableTfidf, setEnableTfidf] = useState(false);
-  const [minClusterSize, setMinClusterSize] = useState(15);
-  const [nNeighbors, setNNeighbors] = useState(15);
-  const [minDist, setMinDist] = useState(0.1);
-
-  const columns = data?.columns ?? [];
-  const results: AnalysisResponse | null = analysisResults;
-
-  const toggleColumn = (col: string) => {
-    setSelected((prev) =>
-      prev.includes(col) ? prev.filter((c) => c !== col) : [...prev, col]
-    );
-  };
-
-  const runAnalysis = async () => {
-    if (selected.length === 0) return;
-    setError(null);
-    setAnalysisRunning(true);
-    try {
-      const res = await api.runAnalysis(selected, {
-        enable_tfidf: enableTfidf,
-        min_cluster_size: minClusterSize,
-        n_neighbors: nNeighbors,
-        min_dist: minDist,
-      });
-      setAnalysisResults(res);
-    } catch (e: unknown) {
-      setError(e instanceof Error ? e.message : 'Analysis failed');
-      setAnalysisRunning(false);
-    }
-  };
-
-  if (!dataLoaded) {
-    return (
-      <Panel title="No Data Loaded">
-        <p className="text-sm text-text-muted">
-          Load a dataset first from the{' '}
-          <button onClick={() => setPage('dashboard')} className="text-accent hover:underline">
-            Dashboard
-          </button>{' '}
-          or{' '}
-          <button onClick={() => setPage('data')} className="text-accent hover:underline">
-            Data Explorer
-          </button>.
-        </p>
-      </Panel>
-    );
-  }
-
-  const tabs: { id: TabId; label: string; icon: typeof Layers; needsResults: boolean }[] = [
-    { id: 'clusters', label: 'Clusters', icon: Layers, needsResults: true },
-    { id: 'correlation', label: 'Correlation', icon: Grid3x3, needsResults: true },
-    { id: 'xgboost', label: 'Feature Importance', icon: BarChart3, needsResults: true },
-    { id: 'distribution', label: 'Distribution', icon: BarChart3, needsResults: true },
-    { id: 'association', label: "Cramér's V Explorer", icon: Network, needsResults: false },
-  ];
-
-  return (
-    <div className="space-y-4">
-      {/* Column selector */}
-      <Panel
-        title="Select Columns for Analysis"
-        subtitle="Choose columns to run through the clustering and ML pipeline"
-        actions={
-          <div className="flex items-center gap-2">
-            <button
-              onClick={() => setShowParams((s) => !s)}
-              className={`flex items-center gap-1.5 rounded-md border px-3 py-1.5 text-xs transition-colors ${
-                showParams ? 'border-accent text-accent' : 'border-border text-text-secondary hover:text-text-primary'
-              }`}
-            >
-              <Settings2 className="h-3.5 w-3.5" /> Params
-            </button>
-            <button
-              onClick={runAnalysis}
-              disabled={selected.length === 0 || analysisRunning}
-              className="flex items-center gap-2 rounded-md bg-accent-dim px-4 py-1.5 text-xs font-medium text-white transition-colors hover:bg-accent disabled:opacity-50"
-            >
-              <Play className="h-3.5 w-3.5" />
-              {analysisRunning ? 'Running...' : 'Run Analysis'}
-            </button>
-          </div>
-        }
-      >
-        <div className="flex flex-wrap gap-2">
-          {columns.map((col) => (
-            <button
-              key={col}
-              onClick={() => toggleColumn(col)}
-              className={`rounded-md border px-3 py-1.5 text-xs transition-colors ${
-                selected.includes(col)
-                  ? 'border-accent bg-accent-dim/30 text-accent-bright'
-                  : 'border-border bg-raised text-text-secondary hover:border-border-bright'
-              }`}
-            >
-              {col}
-            </button>
-          ))}
-        </div>
-        {selected.length > 0 && (
-          <p className="mt-2 text-xs text-text-muted">
-            {selected.length} column{selected.length > 1 ? 's' : ''} selected
-          </p>
-        )}
-
-        {showParams && (
-          <div className="mt-3 grid grid-cols-1 gap-4 rounded-md border border-border/50 bg-raised/50 p-3 sm:grid-cols-2 lg:grid-cols-4">
-            <label className="flex items-center gap-2 text-xs text-text-secondary">
-              <input
-                type="checkbox"
-                checked={enableTfidf}
-                onChange={(e) => setEnableTfidf(e.target.checked)}
-                className="accent-accent"
-              />
-              TF-IDF cluster naming + merging
-            </label>
-            <div>
-              <label className="mb-1 block text-[11px] text-text-muted">
-                Min cluster size: {minClusterSize}
-              </label>
-              <input
-                type="range" min={2} max={50} value={minClusterSize}
-                onChange={(e) => setMinClusterSize(Number(e.target.value))}
-                className="w-full accent-accent"
-              />
-            </div>
-            <div>
-              <label className="mb-1 block text-[11px] text-text-muted">
-                UMAP neighbors: {nNeighbors}
-              </label>
-              <input
-                type="range" min={2} max={100} value={nNeighbors}
-                onChange={(e) => setNNeighbors(Number(e.target.value))}
-                className="w-full accent-accent"
-              />
-            </div>
-            <div>
-              <label className="mb-1 block text-[11px] text-text-muted">
-                UMAP min dist: {minDist.toFixed(2)}
-              </label>
-              <input
-                type="range" min={0} max={1} step={0.05} value={minDist}
-                onChange={(e) => setMinDist(Number(e.target.value))}
-                className="w-full accent-accent"
-              />
-            </div>
-          </div>
-        )}
-      </Panel>
-
-      {analysisRunning && <LoadingSpinner text="Running analysis pipeline..." />}
-
-      {error && (
-        <div className="flex items-center gap-2 rounded-md border border-danger/30 bg-danger/10 px-4 py-2.5 text-sm text-danger">
-          <AlertTriangle className="h-4 w-4" /> {error}
-        </div>
-      )}
-
-      {/* Success message */}
-      {results && (
-        <>
-          <div className="flex items-center gap-2 rounded-md border border-success/30 bg-success/10 px-4 py-2.5 text-sm text-success">
-            <CheckCircle2 className="h-4 w-4" />
-            Analysis complete for {Object.keys(results.results).length} column(s)
-          </div>
-          {results.mock_mode && (
-            <div className="flex items-center gap-2 rounded-md border border-warning/30 bg-warning/10 px-4 py-2.5 text-sm text-warning">
-              <AlertTriangle className="h-4 w-4" />
-              Mock analysis mode is enabled. Outputs are for demo/debugging, not scientific conclusions.
-            </div>
-          )}
-        </>
-      )}
-
-      {/* Tab bar — always available; the Cramér's V explorer needs no analysis run */}
-      <div className="flex flex-wrap gap-1 border-b border-border">
-        {tabs.map(({ id, label, icon: Icon }) => (
-          <button
-            key={id}
-            onClick={() => setActiveTab(id)}
-            className={`flex items-center gap-1.5 border-b-2 px-4 py-2.5 text-xs font-medium transition-colors ${
-              activeTab === id
-                ? 'border-accent text-accent'
-                : 'border-transparent text-text-muted hover:text-text-secondary'
-            }`}
-          >
-            <Icon className="h-3.5 w-3.5" />
-            {label}
-          </button>
-        ))}
-      </div>
-
-      {/* Association explorer — independent of the cluster pipeline. Its XGBoost
-          run is handed to the Feature Importance tab via handleAssocXgboost. */}
-      {activeTab === 'association' && (
-        <CramersVExplorer source="dataset" onXgboost={handleAssocXgboost} />
-      )}
-
-      {/* Feature importance — independent of the cluster pipeline: it shows the
-          explorer-driven results when present, otherwise the pipeline's. */}
-      {activeTab === 'xgboost' && (() => {
-        const xgb =
-          assocXgboost && Object.keys(assocXgboost).length > 0
-            ? assocXgboost
-            : results?.xgboost ?? null;
-        if (xgb && Object.keys(xgb).length > 0) {
-          return (
-            <div className="space-y-3">
-              {assocXgboost && Object.keys(assocXgboost).length > 0 && (
-                <div className="flex items-center gap-2 rounded-md border border-purple/30 bg-purple/10 px-4 py-2.5 text-xs text-text-secondary">
-                  <Network className="h-4 w-4 text-purple" />
-                  Computed directly from your Cramér's V column selection — each column predicted
-                  from the others.
-                </div>
-              )}
-              <XGBoostResults results={xgb} />
-            </div>
-          );
-        }
-        return (
-          <Panel title="Feature Importance">
-            <p className="text-sm text-text-muted">
-              No feature-importance results yet. Run the cluster pipeline above, or open the{' '}
-              <button onClick={() => setActiveTab('association')} className="text-accent hover:underline">
-                Cramér's V Explorer
-              </button>
-              , select columns, and click <span className="text-accent">Feature Importance →</span>.
-            </p>
-          </Panel>
-        );
-      })()}
-
-      {/* Result-dependent tabs */}
-      {activeTab !== 'association' && activeTab !== 'xgboost' && !results && (
-        <Panel title="Run the analysis pipeline">
-          <p className="text-sm text-text-muted">
-            Select columns above and click <span className="text-accent">Run Analysis</span> to
-            populate clustering, correlation, feature-importance and distribution views. The{' '}
-            <button onClick={() => setActiveTab('association')} className="text-accent hover:underline">
-              Cramér's V Explorer
-            </button>{' '}
-            works directly on raw columns without running the pipeline.
-          </p>
-        </Panel>
-      )}
-
-      {results && (
-        <>
-          {/* Cluster visualizations */}
-          {activeTab === 'clusters' && (
-            <div className="grid grid-cols-1 gap-4 xl:grid-cols-2">
-              {Object.entries(results.cluster_viz).map(([col, viz]) => (
-                <Panel key={col} title={viz.title}>
-                  <ClusterVisualization viz={viz} />
-                </Panel>
-              ))}
-            </div>
-          )}
-
-          {/* Correlation heatmap */}
-          {activeTab === 'correlation' && results.cramers_v && (
-            <Panel title="Cramer's V Correlation Matrix">
-              <CorrelationHeatmap data={results.cramers_v} height={500} />
-            </Panel>
-          )}
-          {activeTab === 'correlation' && !results.cramers_v && (
-            <Panel title="Correlation">
-              <p className="text-sm text-text-muted">
-                Select at least 2 columns to compute correlation analysis.
-              </p>
-            </Panel>
-          )}
-
-          {/* Distribution */}
-          {activeTab === 'distribution' && (
-            <div className="grid grid-cols-1 gap-4 lg:grid-cols-2 xl:grid-cols-3">
-              {Object.entries(results.results).map(([col, analysis]) => (
-                <Panel key={col} title={`${col} (${analysis.cluster_count} clusters)`}>
-                  <DistributionChart
-                    data={analysis.distribution}
-                    title={col}
-                    height={280}
-                  />
-                </Panel>
-              ))}
-            </div>
-          )}
-        </>
-      )}
-    </div>
-  );
-}

frontend/src/components/analysis/ClusterView.tsx

@@ -1,21 +0,0 @@
-import { Panel } from '../common/Panel';
-
-export function ClusterView() {
-    return (
-        <div className="h-full space-y-4">
-            <Panel
-                title="EDA Clusters (LLM Analysis)"
-                subtitle="Advanced clustering of UAP sightings using SentenceTransformers and HDBSCAN"
-                noPad
-            >
-                <div className="relative h-[calc(100vh-180px)] w-full overflow-hidden rounded-b-lg">
-                    <iframe
-                        src="/api/analysis/clusters"
-                        className="h-full w-full border-0"
-                        title="UAP Clusters LLM"
-                    />
-                </div>
-            </Panel>
-        </div>
-    );
-}

frontend/src/components/analysis/ClusterVisualization.tsx

@@ -1,64 +0,0 @@
-import Plot from 'react-plotly.js';
-import type { ClusterViz } from '../../types';
-
-const COLORS = [
-  '#58a6ff', '#3fb950', '#f0883e', '#bc8cff', '#39d2c0',
-  '#f85149', '#d29922', '#79c0ff', '#56d364', '#ffa657',
-  '#d2a8ff', '#a5d6ff', '#7ee787', '#ffd8b5', '#e2c5ff',
-  '#76e3ea', '#ff7b72', '#e3b341', '#87ceeb', '#ff69b4',
-];
-
-interface Props {
-  viz: ClusterViz;
-  height?: number;
-}
-
-export function ClusterVisualization({ viz, height = 450 }: Props) {
-  const traces = viz.traces.map((t, i) => ({
-    x: t.x,
-    y: t.y,
-    text: t.text,
-    name: `${t.name} (${t.count})`,
-    type: 'scatter' as const,
-    mode: 'markers' as const,
-    marker: {
-      color: COLORS[i % COLORS.length],
-      size: 5,
-      opacity: 0.8,
-    },
-    hoverinfo: 'text' as const,
-  }));
-
-  return (
-    <Plot
-      data={traces}
-      layout={{
-        title: { text: viz.title, font: { color: '#e6edf3', size: 14 } },
-        paper_bgcolor: 'transparent',
-        plot_bgcolor: '#111820',
-        font: { color: '#8b949e', size: 10 },
-        margin: { l: 40, r: 20, t: 40, b: 40 },
-        xaxis: {
-          gridcolor: '#21283b',
-          zerolinecolor: '#30363d',
-          showticklabels: false,
-        },
-        yaxis: {
-          gridcolor: '#21283b',
-          zerolinecolor: '#30363d',
-          showticklabels: false,
-        },
-        legend: {
-          font: { size: 9, color: '#8b949e' },
-          bgcolor: 'transparent',
-          x: 1.02,
-          y: 1,
-        },
-        height,
-        showlegend: true,
-      }}
-      config={{ responsive: true, displayModeBar: true, displaylogo: false }}
-      style={{ width: '100%' }}
-    />
-  );
-}

frontend/src/components/analysis/CorrelationHeatmap.tsx

@@ -1,70 +0,0 @@
-import Plot from 'react-plotly.js';
-import type { CramersVData } from '../../types';
-
-interface Props {
-  data: CramersVData;
-  height?: number;
-}
-
-export function CorrelationHeatmap({ data, height = 400 }: Props) {
-  // Mask upper triangle
-  const masked = data.matrix.map((row, i) =>
-    row.map((val, j) => (j > i ? null : val))
-  );
-
-  // Annotation text
-  const annotations = [];
-  for (let i = 0; i < data.labels.length; i++) {
-    for (let j = 0; j <= i; j++) {
-      annotations.push({
-        x: data.labels[j],
-        y: data.labels[i],
-        text: masked[i][j] != null ? masked[i][j]!.toFixed(2) : '',
-        font: { color: '#e6edf3', size: 10 },
-        showarrow: false,
-      });
-    }
-  }
-
-  return (
-    <Plot
-      data={[
-        {
-          z: masked,
-          x: data.labels,
-          y: data.labels,
-          type: 'heatmap',
-          colorscale: [
-            [0, '#0d1117'],
-            [0.25, '#1f3a5f'],
-            [0.5, '#3d6098'],
-            [0.75, '#d29922'],
-            [1, '#f85149'],
-          ],
-          zmin: 0,
-          zmax: 1,
-          hoverongaps: false,
-          colorbar: {
-            title: { text: "Cramer's V", font: { color: '#8b949e', size: 10 } },
-            tickfont: { color: '#8b949e', size: 9 },
-          },
-        },
-      ]}
-      layout={{
-        title: {
-          text: "Cramer's V Correlation Matrix",
-          font: { color: '#e6edf3', size: 14 },
-        },
-        paper_bgcolor: 'transparent',
-        plot_bgcolor: 'transparent',
-        font: { color: '#8b949e', size: 10 },
-        margin: { l: 100, r: 40, t: 40, b: 100 },
-        xaxis: { tickangle: -45 },
-        annotations,
-        height,
-      }}
-      config={{ responsive: true, displayModeBar: false }}
-      style={{ width: '100%' }}
-    />
-  );
-}

frontend/src/components/analysis/CramersVExplorer.tsx

@@ -1,487 +0,0 @@
-import { useState, useEffect, useMemo } from 'react';
-import Plot from 'react-plotly.js';
-import { Play, AlertTriangle, Grid3x3, Layers, BarChart3 } from 'lucide-react';
-import { api } from '../../api/client';
-import { Panel } from '../common/Panel';
-import { LoadingSpinner } from '../common/LoadingSpinner';
-import { XGBoostResults } from './XGBoostResults';
-import type { CramersVResponse, ContingencyResponse, ColumnGroup, XGBoostResult } from '../../types';
-
-interface Props {
-  source: 'dataset' | 'parsed';
-  // When provided, XGBoost feature importance computed from the selected columns
-  // is handed off to the parent (e.g. the Feature Importance tab) instead of
-  // rendering inline.
-  onXgboost?: (results: Record<string, XGBoostResult>) => void;
-}
-
-export function CramersVExplorer({ source, onXgboost }: Props) {
-  const [report, setReport] = useState<CramersVResponse | null>(null);
-  const [contingency, setContingency] = useState<ContingencyResponse | null>(null);
-  const [pair, setPair] = useState<{ a: string; b: string } | null>(null);
-
-  const [dropMissing, setDropMissing] = useState(false);
-  const [excludeTrivial, setExcludeTrivial] = useState(true);
-  const [strong, setStrong] = useState(0.3);
-
-  const [loading, setLoading] = useState(false);
-  const [error, setError] = useState<string | null>(null);
-
-  // XGBoost feature importance run directly on the selected columns.
-  const [xgbLoading, setXgbLoading] = useState(false);
-  const [localXgb, setLocalXgb] = useState<Record<string, XGBoostResult> | null>(null);
-
-  // Eligible categorical columns, grouped by their dotted parent (e.g. craft.*).
-  const [groups, setGroups] = useState<ColumnGroup[]>([]);
-  const [selected, setSelected] = useState<Set<string>>(new Set());
-  const [groupsError, setGroupsError] = useState<string | null>(null);
-
-  // Load the parent groups up front — cheap (cardinality only, no matrix), so the
-  // selector is usable before the first Compute. Defaults to all eligible columns,
-  // which matches the explorer's prior "compute everything" behavior.
-  useEffect(() => {
-    let cancelled = false;
-    setGroupsError(null);
-    api
-      .columnGroups({ source })
-      .then((res) => {
-        if (cancelled) return;
-        setGroups(res.groups);
-        setSelected(new Set(res.eligible));
-      })
-      .catch((e) => {
-        if (!cancelled) setGroupsError(e instanceof Error ? e.message : 'Could not load columns');
-      });
-    return () => {
-      cancelled = true;
-    };
-  }, [source]);
-
-  // Flattened in grouped order so the matrix keeps related columns adjacent.
-  const orderedEligible = useMemo(() => groups.flatMap((g) => g.columns), [groups]);
-  const nestedGroups = groups.filter((g) => g.nested);
-  const standaloneCols = groups.filter((g) => !g.nested).flatMap((g) => g.columns);
-
-  const toggleCol = (c: string) =>
-    setSelected((prev) => {
-      const next = new Set(prev);
-      if (next.has(c)) next.delete(c);
-      else next.add(c);
-      return next;
-    });
-
-  const toggleGroup = (g: ColumnGroup) =>
-    setSelected((prev) => {
-      const next = new Set(prev);
-      const allOn = g.columns.every((c) => next.has(c));
-      for (const c of g.columns) {
-        if (allOn) next.delete(c);
-        else next.add(c);
-      }
-      return next;
-    });
-
-  const selectAll = () => setSelected(new Set(orderedEligible));
-  const clearAll = () => setSelected(new Set());
-
-  const run = async () => {
-    if (orderedEligible.length && selected.size < 2) {
-      setError('Select at least two columns (or whole parent groups) to compute associations.');
-      return;
-    }
-    setLoading(true);
-    setError(null);
-    setContingency(null);
-    setPair(null);
-    try {
-      const cols = orderedEligible.filter((c) => selected.has(c));
-      const res = await api.cramersV({
-        source,
-        columns: cols.length ? cols : undefined,
-        drop_missing: dropMissing,
-        exclude_trivial: excludeTrivial,
-        strong_threshold: strong,
-      });
-      setReport(res);
-    } catch (e) {
-      setError(e instanceof Error ? e.message : 'Cramér’s V failed');
-    } finally {
-      setLoading(false);
-    }
-  };
-
-  const runXgboost = async () => {
-    const cols = orderedEligible.filter((c) => selected.has(c));
-    if (cols.length < 2) {
-      setError('Select at least two columns to run feature importance.');
-      return;
-    }
-    setXgbLoading(true);
-    setError(null);
-    try {
-      const res = await api.xgboostImportance(cols, source);
-      if (!Object.keys(res.results).length) {
-        setError(res.message || 'No feature-importance results (need ≥2 non-constant columns).');
-        return;
-      }
-      if (onXgboost) onXgboost(res.results);
-      else setLocalXgb(res.results);
-    } catch (e) {
-      setError(e instanceof Error ? e.message : 'Feature importance failed');
-    } finally {
-      setXgbLoading(false);
-    }
-  };
-
-  const loadContingency = async (a: string, b: string) => {
-    setPair({ a, b });
-    try {
-      const res = await api.contingency({ col1: a, col2: b, drop_missing: dropMissing, source });
-      setContingency(res);
-    } catch (e) {
-      setError(e instanceof Error ? e.message : 'Contingency failed');
-    }
-  };
-
-  // Lower-triangle masked matrix for the heatmap
-  const masked = report
-    ? report.matrix.map((row, i) => row.map((val, j) => (j > i ? null : val)))
-    : [];
-
-  return (
-    <div className="space-y-4">
-      <Panel
-        title="Categorical Association Explorer (Cramér's V)"
-        subtitle="Pairwise association across the selected categorical columns"
-        actions={
-          <div className="flex items-center gap-2">
-            <button
-              onClick={runXgboost}
-              disabled={xgbLoading || selected.size < 2}
-              title="Train XGBoost on the selected columns and send the result to Feature Importance"
-              className="flex items-center gap-1.5 rounded-md border border-border bg-raised px-3 py-1.5 text-xs font-medium text-text-secondary transition-colors hover:border-accent hover:text-accent disabled:opacity-50"
-            >
-              <BarChart3 className="h-3.5 w-3.5" />
-              {xgbLoading ? 'Training…' : 'Feature Importance →'}
-            </button>
-            <button
-              onClick={run}
-              disabled={loading}
-              className="flex items-center gap-2 rounded-md bg-accent-dim px-4 py-1.5 text-xs font-medium text-white transition-colors hover:bg-accent disabled:opacity-50"
-            >
-              <Play className="h-3.5 w-3.5" />
-              {loading ? 'Computing…' : 'Compute'}
-            </button>
-          </div>
-        }
-      >
-        <div className="flex flex-wrap items-center gap-4">
-          <label className="flex items-center gap-2 text-xs text-text-secondary">
-            <input
-              type="checkbox"
-              checked={dropMissing}
-              onChange={(e) => setDropMissing(e.target.checked)}
-              className="accent-accent"
-            />
-            Drop missing (complete-case pairs)
-          </label>
-          <label className="flex items-center gap-2 text-xs text-text-secondary">
-            <input
-              type="checkbox"
-              checked={excludeTrivial}
-              onChange={(e) => setExcludeTrivial(e.target.checked)}
-              className="accent-accent"
-            />
-            Exclude trivial (V≈0 / V≈1)
-          </label>
-          <label className="flex items-center gap-2 text-xs text-text-secondary">
-            Strong ≥ {strong.toFixed(2)}
-            <input
-              type="range"
-              min={0.1}
-              max={0.9}
-              step={0.05}
-              value={strong}
-              onChange={(e) => setStrong(Number(e.target.value))}
-              className="w-28 accent-accent"
-            />
-          </label>
-        </div>
-      </Panel>
-
-      {/* Column / parent-group selector */}
-      <Panel
-        title="Columns"
-        subtitle="Add whole parent groups (nested dot-separated names) or individual columns"
-        actions={
-          <div className="flex items-center gap-2 text-[11px]">
-            <span className="text-text-muted">
-              {selected.size}/{orderedEligible.length} selected
-            </span>
-            <button
-              onClick={selectAll}
-              className="rounded border border-border px-2 py-0.5 text-text-secondary transition-colors hover:border-accent hover:text-accent"
-            >
-              All
-            </button>
-            <button
-              onClick={clearAll}
-              className="rounded border border-border px-2 py-0.5 text-text-secondary transition-colors hover:border-accent hover:text-accent"
-            >
-              Clear
-            </button>
-          </div>
-        }
-      >
-        {groupsError && <p className="text-xs text-danger">{groupsError}</p>}
-        {!groupsError && orderedEligible.length === 0 && (
-          <p className="text-xs text-text-muted">
-            No categorical-eligible columns found for this source (binary/low/medium cardinality).
-          </p>
-        )}
-
-        <div className="space-y-3">
-          {nestedGroups.map((g) => {
-            const sel = g.columns.filter((c) => selected.has(c)).length;
-            const all = sel === g.columns.length;
-            return (
-              <div key={g.parent} className="rounded-md border border-border/50 bg-raised/40 p-2">
-                <button
-                  onClick={() => toggleGroup(g)}
-                  title={all ? 'Remove whole group' : 'Add whole group'}
-                  className={`mb-1.5 flex items-center gap-1.5 rounded px-1.5 py-0.5 text-xs font-semibold transition-colors ${
-                    all ? 'text-accent-bright' : sel ? 'text-accent' : 'text-text-secondary hover:text-text-primary'
-                  }`}
-                >
-                  <Layers className="h-3.5 w-3.5" />
-                  {g.parent}
-                  <span className="font-normal text-text-muted">
-                    ({sel}/{g.columns.length})
-                  </span>
-                </button>
-                <div className="flex flex-wrap gap-1.5 pl-1">
-                  {g.columns.map((c, i) => (
-                    <button
-                      key={c}
-                      onClick={() => toggleCol(c)}
-                      title={c}
-                      className={`rounded-md border px-2 py-1 text-[11px] transition-colors ${
-                        selected.has(c)
-                          ? 'border-accent bg-accent-dim/30 text-accent-bright'
-                          : 'border-border bg-raised text-text-secondary hover:border-border-bright'
-                      }`}
-                    >
-                      {g.leaves[i]}
-                    </button>
-                  ))}
-                </div>
-              </div>
-            );
-          })}
-
-          {standaloneCols.length > 0 && (
-            <div>
-              {nestedGroups.length > 0 && (
-                <p className="mb-1.5 text-[11px] font-medium text-text-muted">Ungrouped columns</p>
-              )}
-              <div className="flex flex-wrap gap-1.5">
-                {standaloneCols.map((c) => (
-                  <button
-                    key={c}
-                    onClick={() => toggleCol(c)}
-                    className={`rounded-md border px-2 py-1 text-[11px] transition-colors ${
-                      selected.has(c)
-                        ? 'border-accent bg-accent-dim/30 text-accent-bright'
-                        : 'border-border bg-raised text-text-secondary hover:border-border-bright'
-                    }`}
-                  >
-                    {c}
-                  </button>
-                ))}
-              </div>
-            </div>
-          )}
-        </div>
-      </Panel>
-
-      {error && (
-        <div className="flex items-center gap-2 rounded-md border border-danger/30 bg-danger/10 px-4 py-2.5 text-sm text-danger">
-          <AlertTriangle className="h-4 w-4" /> {error}
-        </div>
-      )}
-
-      {loading && <LoadingSpinner text="Computing Cramér's V matrix..." />}
-      {xgbLoading && <LoadingSpinner text="Training XGBoost on the selected columns..." />}
-
-      {localXgb && (
-        <Panel
-          title="Feature Importance (XGBoost)"
-          subtitle="Each selected column predicted from the others — gain-based importance"
-        >
-          <XGBoostResults results={localXgb} />
-        </Panel>
-      )}
-
-      {report && report.labels.length < 2 && (
-        <Panel title="Not enough categorical columns">
-          <p className="text-sm text-text-muted">
-            Fewer than two suitable categorical columns were selected (binary/low/medium cardinality).
-            High-cardinality, free-text and constant columns are excluded automatically.
-          </p>
-        </Panel>
-      )}
-
-      {report && report.labels.length >= 2 && (
-        <div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
-          {/* Heatmap */}
-          <div className="xl:col-span-2">
-            <Panel title="Association Matrix" subtitle="Click a cell to drill into the contingency table" noPad>
-              <div className="p-2">
-                <Plot
-                  data={[
-                    {
-                      z: masked,
-                      x: report.labels,
-                      y: report.labels,
-                      type: 'heatmap',
-                      colorscale: [
-                        [0, '#0d1117'],
-                        [0.25, '#1f3a5f'],
-                        [0.5, '#3d6098'],
-                        [0.75, '#d29922'],
-                        [1, '#f85149'],
-                      ],
-                      zmin: 0,
-                      zmax: 1,
-                      hoverongaps: false,
-                      colorbar: {
-                        title: { text: "Cramér's V", font: { color: '#8b949e', size: 10 } },
-                        tickfont: { color: '#8b949e', size: 9 },
-                      },
-                    },
-                  ]}
-                  layout={{
-                    paper_bgcolor: 'transparent',
-                    plot_bgcolor: 'transparent',
-                    font: { color: '#8b949e', size: 9 },
-                    margin: { l: 130, r: 30, t: 20, b: 130 },
-                    xaxis: { tickangle: -45, automargin: true },
-                    yaxis: { automargin: true },
-                    height: Math.max(360, report.labels.length * 26),
-                  }}
-                  config={{ responsive: true, displayModeBar: false }}
-                  style={{ width: '100%' }}
-                  onClick={(e: Readonly<{ points?: Array<{ x?: unknown; y?: unknown }> }>) => {
-                    const pt = e.points?.[0];
-                    if (pt && pt.x != null && pt.y != null) {
-                      loadContingency(String(pt.y), String(pt.x));
-                    }
-                  }}
-                />
-              </div>
-            </Panel>
-
-            {/* Contingency drilldown */}
-            {pair && contingency && (
-              <Panel
-                title={`Contingency: ${pair.a} × ${pair.b}`}
-                subtitle={`Cramér's V = ${contingency.v} · N = ${contingency.n.toLocaleString()}`}
-                className="mt-4"
-                noPad
-              >
-                <div className="overflow-auto p-2" style={{ maxHeight: 360 }}>
-                  <table className="border-collapse text-[11px]">
-                    <thead>
-                      <tr>
-                        <th className="sticky left-0 bg-deep px-2 py-1 text-left text-text-muted">
-                          {pair.a} \ {pair.b}
-                        </th>
-                        {contingency.col_labels.map((c) => (
-                          <th key={c} className="px-2 py-1 text-text-muted" title={c}>
-                            <div className="max-w-24 truncate">{c}</div>
-                          </th>
-                        ))}
-                      </tr>
-                    </thead>
-                    <tbody>
-                      {contingency.row_labels.map((r, i) => (
-                        <tr key={r} className="border-t border-border/30">
-                          <td className="sticky left-0 bg-surface px-2 py-1 font-medium text-text-secondary" title={r}>
-                            <div className="max-w-32 truncate">{r}</div>
-                          </td>
-                          {contingency.matrix[i].map((v, j) => (
-                            <td key={j} className="px-2 py-1 text-center text-text-secondary">
-                              {v || ''}
-                            </td>
-                          ))}
-                        </tr>
-                      ))}
-                    </tbody>
-                  </table>
-                </div>
-              </Panel>
-            )}
-          </div>
-
-          {/* Pairs + high-corr columns */}
-          <div className="space-y-4">
-            <Panel
-              title="Strongest Pairs"
-              subtitle={report.n_excluded ? `${report.n_excluded} trivial pairs hidden` : undefined}
-            >
-              <div className="max-h-96 space-y-1 overflow-y-auto">
-                {report.pairs.slice(0, 40).map((p, i) => (
-                  <button
-                    key={i}
-                    onClick={() => loadContingency(p.a, p.b)}
-                    className="flex w-full items-center justify-between gap-2 rounded border border-border/40 bg-raised px-2.5 py-1.5 text-left text-[11px] hover:border-accent"
-                  >
-                    <span className="truncate text-text-secondary">
-                      {p.a} <span className="text-text-muted">×</span> {p.b}
-                    </span>
-                    <span
-                      className={`shrink-0 font-mono font-semibold ${
-                        p.v >= strong ? 'text-accent-bright' : 'text-text-muted'
-                      }`}
-                    >
-                      {p.v.toFixed(3)}
-                    </span>
-                  </button>
-                ))}
-                {report.pairs.length === 0 && (
-                  <p className="text-xs text-text-muted">No non-trivial pairs found.</p>
-                )}
-              </div>
-            </Panel>
-
-            {report.high_correlation_columns.length > 0 && (
-              <Panel title={`High-Correlation Columns (≥ ${strong.toFixed(2)})`}>
-                <div className="flex flex-wrap gap-1.5">
-                  {report.high_correlation_columns.map((c) => (
-                    <span
-                      key={c}
-                      className="rounded-md border border-purple/40 bg-purple/10 px-2 py-1 text-[11px] text-text-primary"
-                    >
-                      {c}
-                    </span>
-                  ))}
-                </div>
-              </Panel>
-            )}
-          </div>
-        </div>
-      )}
-
-      {!report && !loading && (
-        <Panel title="Cramér's V">
-          <p className="flex items-center gap-2 text-sm text-text-muted">
-            <Grid3x3 className="h-4 w-4" />
-            Pick parent groups / columns above, then click{' '}
-            <span className="text-accent">Compute</span> to score categorical associations
-            across the {source === 'parsed' ? 'parsed' : 'loaded'} dataset.
-          </p>
-        </Panel>
-      )}
-    </div>
-  );
-}

frontend/src/components/analysis/DistributionChart.tsx

@@ -1,43 +0,0 @@
-import Plot from 'react-plotly.js';
-import type { DistributionItem } from '../../types';
-
-interface Props {
-  data: DistributionItem[];
-  title: string;
-  height?: number;
-}
-
-const COLORS = [
-  '#58a6ff', '#3fb950', '#f0883e', '#bc8cff', '#39d2c0',
-  '#f85149', '#d29922', '#79c0ff', '#56d364', '#ffa657',
-];
-
-export function DistributionChart({ data, title, height = 300 }: Props) {
-  return (
-    <Plot
-      data={[
-        {
-          labels: data.map((d) => d.label),
-          values: data.map((d) => d.count),
-          type: 'pie',
-          hole: 0.5,
-          marker: { colors: COLORS },
-          textfont: { color: '#e6edf3', size: 9 },
-          hovertemplate: '%{label}<br>Count: %{value}<br>%{percent}<extra></extra>',
-        },
-      ]}
-      layout={{
-        title: { text: title, font: { color: '#e6edf3', size: 13 } },
-        paper_bgcolor: 'transparent',
-        plot_bgcolor: 'transparent',
-        font: { color: '#8b949e', size: 9 },
-        margin: { l: 10, r: 10, t: 40, b: 10 },
-        showlegend: true,
-        legend: { font: { size: 8, color: '#8b949e' }, bgcolor: 'transparent' },
-        height,
-      }}
-      config={{ responsive: true, displayModeBar: false }}
-      style={{ width: '100%' }}
-    />
-  );
-}

frontend/src/components/analysis/XGBoostResults.tsx

@@ -1,79 +0,0 @@
-import Plot from 'react-plotly.js';
-import type { XGBoostResult } from '../../types';
-
-interface Props {
-  results: Record<string, XGBoostResult>;
-}
-
-export function XGBoostResults({ results }: Props) {
-  const columns = Object.keys(results);
-
-  return (
-    <div className="space-y-4">
-      {columns.map((col) => {
-        const r = results[col];
-        const features = Object.keys(r.feature_importance);
-        const importances = Object.values(r.feature_importance);
-
-        return (
-          <div key={col} className="rounded-lg border border-border bg-surface">
-            <div className="flex items-center justify-between border-b border-border px-4 py-3">
-              <div>
-                <h4 className="text-sm font-semibold text-text-primary">{col}</h4>
-                <p className="text-xs text-text-muted">Feature importance analysis</p>
-              </div>
-              <div className="flex items-center gap-2">
-                <span className="text-xs text-text-muted">Accuracy:</span>
-                <span
-                  className={`rounded-full px-2.5 py-0.5 text-xs font-bold ${
-                    r.accuracy >= 0.8
-                      ? 'bg-success/20 text-success'
-                      : r.accuracy >= 0.6
-                        ? 'bg-warning/20 text-warning'
-                        : 'bg-danger/20 text-danger'
-                  }`}
-                >
-                  {(r.accuracy * 100).toFixed(1)}%
-                </span>
-              </div>
-            </div>
-            <div className="p-4">
-              <Plot
-                data={[
-                  {
-                    y: features,
-                    x: importances,
-                    type: 'bar',
-                    orientation: 'h',
-                    marker: {
-                      color: importances.map((v) => {
-                        const t = v / Math.max(...importances, 0.001);
-                        return `rgba(88, 166, 255, ${0.3 + t * 0.7})`;
-                      }),
-                    },
-                    hovertemplate: '%{y}: %{x:.3f}<extra></extra>',
-                  },
-                ]}
-                layout={{
-                  paper_bgcolor: 'transparent',
-                  plot_bgcolor: 'transparent',
-                  font: { color: '#8b949e', size: 10 },
-                  margin: { l: 120, r: 20, t: 10, b: 30 },
-                  xaxis: {
-                    title: { text: 'Importance (Gain)', font: { size: 10 } },
-                    gridcolor: '#21283b',
-                    zerolinecolor: '#30363d',
-                  },
-                  yaxis: { autorange: 'reversed' },
-                  height: 200,
-                }}
-                config={{ responsive: true, displayModeBar: false }}
-                style={{ width: '100%' }}
-              />
-            </div>
-          </div>
-        );
-      })}
-    </div>
-  );
-}

frontend/src/components/common/LoadingSpinner.tsx

@@ -1,11 +0,0 @@
-export function LoadingSpinner({ text = 'Loading...' }: { text?: string }) {
-  return (
-    <div className="flex flex-col items-center justify-center gap-3 py-12">
-      <div className="relative h-10 w-10">
-        <div className="absolute inset-0 rounded-full border-2 border-border opacity-30" />
-        <div className="absolute inset-0 animate-spin rounded-full border-2 border-transparent border-t-accent" />
-      </div>
-      <span className="text-sm text-text-secondary">{text}</span>
-    </div>
-  );
-}

frontend/src/components/common/Panel.tsx

@@ -1,27 +0,0 @@
-import type { ReactNode } from 'react';
-
-interface PanelProps {
-  title?: string;
-  subtitle?: string;
-  children: ReactNode;
-  className?: string;
-  actions?: ReactNode;
-  noPad?: boolean;
-}
-
-export function Panel({ title, subtitle, children, className = '', actions, noPad }: PanelProps) {
-  return (
-    <div className={`rounded-lg border border-border bg-surface ${className}`}>
-      {(title || actions) && (
-        <div className="flex items-center justify-between border-b border-border px-4 py-3">
-          <div>
-            {title && <h3 className="text-sm font-semibold text-text-primary">{title}</h3>}
-            {subtitle && <p className="mt-0.5 text-xs text-text-muted">{subtitle}</p>}
-          </div>
-          {actions && <div className="flex items-center gap-2">{actions}</div>}
-        </div>
-      )}
-      <div className={noPad ? '' : 'p-4'}>{children}</div>
-    </div>
-  );
-}

frontend/src/components/common/StatusBadge.tsx

@@ -1,22 +0,0 @@
-interface StatusBadgeProps {
-  status: 'success' | 'warning' | 'danger' | 'info' | 'idle';
-  label: string;
-  pulse?: boolean;
-}
-
-const colorMap = {
-  success: 'bg-success/20 text-success border-success/30',
-  warning: 'bg-warning/20 text-warning border-warning/30',
-  danger: 'bg-danger/20 text-danger border-danger/30',
-  info: 'bg-accent/20 text-accent border-accent/30',
-  idle: 'bg-border/50 text-text-muted border-border',
-};
-
-export function StatusBadge({ status, label, pulse }: StatusBadgeProps) {
-  return (
-    <span className={`inline-flex items-center gap-1.5 rounded-full border px-2.5 py-0.5 text-xs font-medium ${colorMap[status]}`}>
-      <span className={`h-1.5 w-1.5 rounded-full bg-current ${pulse ? 'animate-pulse' : ''}`} />
-      {label}
-    </span>
-  );
-}

frontend/src/components/dashboard/Dashboard.tsx

@@ -1,194 +0,0 @@
-import { useEffect, useState, useCallback } from 'react';
-import {
-  Database,
-  Columns3,
-  BrainCircuit,
-  HardDrive,
-  Upload,
-  Play,
-  AlertTriangle,
-} from 'lucide-react';
-import { api } from '../../api/client';
-import { useStore } from '../../store/useStore';
-import { StatCard } from './StatCard';
-import { Panel } from '../common/Panel';
-import { LoadingSpinner } from '../common/LoadingSpinner';
-
-export function Dashboard() {
-  const { summary, setSummary, setPage, dataLoaded, setData } = useStore();
-  const [loading, setLoading] = useState(false);
-  const [error, setError] = useState<string | null>(null);
-
-  const fetchSummary = useCallback(async () => {
-    try {
-      const s = await api.getDashboardSummary();
-      setSummary(s);
-    } catch {
-      // Summary endpoint might not have data yet
-    }
-  }, [setSummary]);
-
-  useEffect(() => {
-    fetchSummary();
-  }, [fetchSummary]);
-
-  const handleLoadData = async (type: 'west' | 'east' = 'west') => {
-    setLoading(true);
-    setError(null);
-    try {
-      const res = await api.loadData(type, 15000);
-      setData(res.data, res.column_stats);
-      await fetchSummary();
-    } catch (e: unknown) {
-      setError(e instanceof Error ? e.message : 'Failed to load data');
-    } finally {
-      setLoading(false);
-    }
-  };
-
-  return (
-    <div className="space-y-6">
-      {/* Stat cards */}
-      <div className="grid grid-cols-1 gap-4 sm:grid-cols-2 lg:grid-cols-4">
-        <StatCard
-          label="Total Records"
-          value={summary?.total_rows?.toLocaleString() ?? '---'}
-          icon={<Database className="h-5 w-5" />}
-          color="text-accent"
-        />
-        <StatCard
-          label="Columns"
-          value={summary?.total_columns ?? '---'}
-          icon={<Columns3 className="h-5 w-5" />}
-          color="text-purple"
-        />
-        <StatCard
-          label="Analysis Runs"
-          value={summary?.analysis_runs ?? 0}
-          icon={<BrainCircuit className="h-5 w-5" />}
-          color="text-cyan"
-        />
-        <StatCard
-          label="Memory Usage"
-          value={summary?.memory_mb ? `${summary.memory_mb} MB` : '---'}
-          icon={<HardDrive className="h-5 w-5" />}
-          color="text-orange"
-        />
-      </div>
-
-      {/* Quick actions & status */}
-      <div className="grid grid-cols-1 gap-6 lg:grid-cols-3">
-        {/* Quick Actions */}
-        <Panel title="Quick Actions" className="lg:col-span-1">
-          <div className="space-y-3">
-            <div className="overflow-hidden rounded-md border border-border bg-raised">
-              <div className="flex items-center gap-2 border-b border-border/50 bg-surface/30 px-3 py-2 text-xs font-semibold uppercase tracking-wider text-text-muted">
-                <Upload className="h-3.5 w-3.5" />
-                Load Parsed Dataset
-              </div>
-              <div className="flex divide-x divide-border/50">
-                <button
-                  onClick={() => handleLoadData('west')}
-                  disabled={loading}
-                  className="flex-1 px-4 py-3 text-sm font-medium transition-colors hover:bg-accent/10 hover:text-accent disabled:opacity-50"
-                >
-                  West
-                </button>
-                <button
-                  onClick={() => handleLoadData('east')}
-                  disabled={loading}
-                  className="flex-1 px-4 py-3 text-sm font-medium transition-colors hover:bg-purple/10 hover:text-purple disabled:opacity-50"
-                >
-                  East
-                </button>
-              </div>
-            </div>
-
-            <button
-              onClick={() => setPage('data')}
-              className="flex w-full items-center gap-3 rounded-md border border-border bg-raised px-4 py-3 text-sm text-text-primary transition-colors hover:border-accent hover:bg-elevated"
-            >
-              <Database className="h-4 w-4 text-purple" />
-              Open Data Explorer
-            </button>
-            <button
-              onClick={() => setPage('analysis')}
-              disabled={!dataLoaded}
-              className="flex w-full items-center gap-3 rounded-md border border-border bg-raised px-4 py-3 text-sm text-text-primary transition-colors hover:border-accent hover:bg-elevated disabled:opacity-50"
-            >
-              <Play className="h-4 w-4 text-cyan" />
-              Run Analysis Pipeline
-            </button>
-          </div>
-          {loading && <LoadingSpinner text="Loading dataset..." />}
-          {error && (
-            <div className="mt-3 flex items-center gap-2 rounded-md bg-danger/10 px-3 py-2 text-sm text-danger">
-              <AlertTriangle className="h-4 w-4" /> {error}
-            </div>
-          )}
-        </Panel>
-
-        {/* Data schema overview */}
-        <Panel title="Data Schema" subtitle={summary?.loaded ? `${summary.total_columns} columns detected` : 'Load data to view schema'} className="lg:col-span-2">
-          {summary?.columns && summary.columns.length > 0 ? (
-            <div className="max-h-72 overflow-y-auto">
-              <table className="w-full text-sm">
-                <thead>
-                  <tr className="border-b border-border text-left text-xs uppercase text-text-muted">
-                    <th className="pb-2 pr-4">Column</th>
-                    <th className="pb-2 pr-4">Type</th>
-                    <th className="pb-2">Nulls</th>
-                  </tr>
-                </thead>
-                <tbody>
-                  {summary.columns.map((col) => (
-                    <tr key={col} className="border-b border-border/50">
-                      <td className="py-1.5 pr-4 font-mono text-xs text-text-primary">{col}</td>
-                      <td className="py-1.5 pr-4">
-                        <span className="rounded bg-elevated px-1.5 py-0.5 text-xs text-text-secondary">
-                          {summary.dtypes?.[col] ?? '?'}
-                        </span>
-                      </td>
-                      <td className="py-1.5 text-xs text-text-muted">
-                        {summary.null_counts?.[col] ?? 0}
-                      </td>
-                    </tr>
-                  ))}
-                </tbody>
-              </table>
-            </div>
-          ) : (
-            <p className="text-sm text-text-muted">No data loaded. Use &quot;Load Parsed Dataset&quot; to begin.</p>
-          )}
-        </Panel>
-      </div>
-
-      {/* System status */}
-      <Panel title="System Pipeline">
-        <div className="mb-3 flex flex-wrap items-center gap-3 text-xs text-text-muted">
-          <span>Mode: {summary?.analysis_mode ?? 'mock'}</span>
-          <span>Last run: {summary?.last_analysis_at ? new Date(summary.last_analysis_at).toLocaleString() : 'N/A'}</span>
-        </div>
-        <div className="flex items-center gap-2">
-          {['Data Ingestion', 'Embedding', 'Dimensionality Reduction', 'Clustering', 'TF-IDF Naming', 'Merge Clusters', 'XGBoost', 'Visualization'].map(
-            (step, i) => (
-              <div key={step} className="flex items-center gap-2">
-                <div
-                  className={`rounded-md border px-3 py-1.5 text-xs font-medium ${i === 0 && dataLoaded
-                    ? 'border-success/30 bg-success/10 text-success'
-                    : 'border-border bg-raised text-text-muted'
-                    }`}
-                >
-                  {step}
-                </div>
-                {i < 7 && (
-                  <div className="h-px w-4 bg-border" />
-                )}
-              </div>
-            )
-          )}
-        </div>
-      </Panel>
-    </div>
-  );
-}

frontend/src/components/dashboard/StatCard.tsx

@@ -1,24 +0,0 @@
-import type { ReactNode } from 'react';
-
-interface StatCardProps {
-  label: string;
-  value: string | number;
-  icon: ReactNode;
-  trend?: string;
-  color?: string;
-}
-
-export function StatCard({ label, value, icon, trend, color = 'text-accent' }: StatCardProps) {
-  return (
-    <div className="rounded-lg border border-border bg-surface p-4">
-      <div className="flex items-start justify-between">
-        <div>
-          <p className="text-xs font-medium uppercase tracking-wider text-text-muted">{label}</p>
-          <p className={`mt-1 text-2xl font-bold ${color}`}>{value}</p>
-          {trend && <p className="mt-1 text-xs text-text-secondary">{trend}</p>}
-        </div>
-        <div className="rounded-md bg-elevated p-2 text-text-muted">{icon}</div>
-      </div>
-    </div>
-  );
-}

frontend/src/components/data/DataExplorer.tsx

@@ -1,152 +0,0 @@
-import { useState, useCallback } from 'react';
-import { Upload, FileSpreadsheet, AlertTriangle } from 'lucide-react';
-import { api } from '../../api/client';
-import { useStore } from '../../store/useStore';
-import { Panel } from '../common/Panel';
-import { LoadingSpinner } from '../common/LoadingSpinner';
-import { DataTable } from './DataTable';
-import { FilterPanel } from './FilterPanel';
-import type { ColumnStat, DataResponse } from '../../types';
-
-export function DataExplorer() {
-  const { data, columnStats, setData, dataLoaded } = useStore();
-  const [loading, setLoading] = useState(false);
-  const [error, setError] = useState<string | null>(null);
-  const [activeData, setActiveData] = useState<DataResponse | null>(null);
-  const [activeStats, setActiveStats] = useState<ColumnStat[]>([]);
-
-  const displayData = activeData ?? data;
-  const displayStats = activeStats.length > 0 ? activeStats : columnStats;
-
-
-  const handleUpload = async (e: React.ChangeEvent<HTMLInputElement>) => {
-    const file = e.target.files?.[0];
-    if (!file) return;
-    setLoading(true);
-    setError(null);
-    try {
-      const res = await api.uploadFile(file);
-      setData(res.data, res.column_stats);
-      setActiveData(null);
-      setActiveStats([]);
-    } catch (err: unknown) {
-      setError(err instanceof Error ? err.message : 'Upload failed');
-    } finally {
-      setLoading(false);
-    }
-  };
-
-  const handleFilter = useCallback(
-    async (
-      filters: {
-        column: string;
-        type: string;
-        values?: string[];
-        min_val?: number;
-        max_val?: number;
-        pattern?: string;
-      }[]
-    ) => {
-      if (filters.length === 0) {
-        setActiveData(null);
-        setActiveStats([]);
-        setError(null);
-        return;
-      }
-      setLoading(true);
-      setError(null);
-      try {
-        const res = await api.filterData(filters);
-        setActiveData(res.data);
-        setActiveStats(res.column_stats);
-      } catch (err: unknown) {
-        setError(err instanceof Error ? err.message : 'Filtering failed');
-      } finally {
-        setLoading(false);
-      }
-    },
-    []
-  );
-
-  return (
-    <div className="space-y-4">
-      <div className="flex flex-wrap items-center gap-3">
-
-
-        <label className="flex cursor-pointer items-center gap-2 rounded-md border border-border bg-surface px-4 py-2 text-sm text-text-primary transition-colors hover:border-purple hover:bg-elevated">
-          <Upload className="h-4 w-4 text-purple" />
-          Upload Files
-          <input type="file" accept=".csv,.xlsx,.xls,.json" onChange={handleUpload} className="hidden" />
-        </label>
-
-        {dataLoaded && displayData && (
-          <div className="ml-auto flex items-center gap-3 text-xs text-text-muted">
-            <FileSpreadsheet className="h-4 w-4" />
-            <span>
-              {displayData.returned_rows.toLocaleString()} / {displayData.total_rows.toLocaleString()} rows
-            </span>
-            <span>{displayData.columns.length} columns</span>
-          </div>
-        )}
-      </div>
-
-      {error && (
-        <div className="flex items-center gap-2 rounded-md border border-danger/30 bg-danger/10 px-4 py-2.5 text-sm text-danger">
-          <AlertTriangle className="h-4 w-4" /> {error}
-        </div>
-      )}
-
-      {loading && <LoadingSpinner text="Processing data..." />}
-
-      {/* Filters */}
-      {dataLoaded && displayStats.length > 0 && (
-        <FilterPanel columns={displayStats} onApply={handleFilter} />
-      )}
-
-      {/* Column Stats */}
-      {dataLoaded && displayStats.length > 0 && (
-        <Panel title="Column Statistics" subtitle={`${displayStats.length} columns`}>
-          <div className="grid grid-cols-2 gap-2 sm:grid-cols-3 lg:grid-cols-4 xl:grid-cols-6">
-            {displayStats.map((col) => (
-              <div
-                key={col.name}
-                className="rounded border border-border/50 bg-raised p-2.5"
-              >
-                <p className="truncate text-xs font-medium text-text-primary" title={col.name}>
-                  {col.name}
-                </p>
-                <p className="mt-0.5 text-[10px] text-text-muted">
-                  {col.dtype} | {col.unique} unique
-                </p>
-                {col.top_values && col.top_values.length > 0 && (
-                  <div className="mt-1.5 space-y-0.5">
-                    {col.top_values.slice(0, 3).map((tv) => (
-                      <div key={tv.value} className="flex items-center gap-1">
-                        <div
-                          className="h-1 rounded-full bg-accent/60"
-                          style={{
-                            width: `${Math.min(100, (tv.count / (col.non_null || 1)) * 100)}%`,
-                          }}
-                        />
-                        <span className="whitespace-nowrap text-[9px] text-text-muted">
-                          {tv.value.slice(0, 15)}
-                        </span>
-                      </div>
-                    ))}
-                  </div>
-                )}
-              </div>
-            ))}
-          </div>
-        </Panel>
-      )}
-
-      {/* Data Table */}
-      {dataLoaded && displayData && (
-        <Panel title="Data View" noPad>
-          <DataTable data={displayData} maxHeight="calc(100vh - 420px)" />
-        </Panel>
-      )}
-    </div>
-  );
-}

frontend/src/components/data/DataTable.tsx

@@ -1,299 +0,0 @@
-import { useState, useMemo, useEffect } from 'react';
-import {
-  ChevronUp,
-  ChevronDown,
-  ChevronsUpDown,
-  Maximize2,
-  Minimize2,
-  Download,
-  Eye,
-  Search,
-  X,
-  FileText
-} from 'lucide-react';
-import type { DataResponse } from '../../types';
-
-interface DataTableProps {
-  data: DataResponse;
-  maxHeight?: string;
-}
-
-type SortDir = 'asc' | 'desc' | null;
-
-export function DataTable({ data, maxHeight = '500px' }: DataTableProps) {
-  const [sortCol, setSortCol] = useState<string | null>(null);
-  const [sortDir, setSortDir] = useState<SortDir>(null);
-  const [page, setPage] = useState(0);
-  const [isFullScreen, setIsFullScreen] = useState(false);
-  const [searchTerm, setSearchTerm] = useState('');
-  const [selectedRow, setSelectedRow] = useState<any | null>(null);
-  const pageSize = 100;
-
-  // Handle ESC key for full screen
-  useEffect(() => {
-    const handleEsc = (e: KeyboardEvent) => {
-      if (e.key === 'Escape' && isFullScreen) setIsFullScreen(false);
-    };
-    window.addEventListener('keydown', handleEsc);
-    return () => window.removeEventListener('keydown', handleEsc);
-  }, [isFullScreen]);
-
-  const handleSort = (col: string) => {
-    if (sortCol === col) {
-      setSortDir((d) => (d === 'asc' ? 'desc' : d === 'desc' ? null : 'asc'));
-      if (sortDir === 'desc') setSortCol(null);
-    } else {
-      setSortCol(col);
-      setSortDir('asc');
-    }
-    setPage(0);
-  };
-
-  const filtered = useMemo(() => {
-    if (!searchTerm) return data.rows;
-    const lower = searchTerm.toLowerCase();
-    return data.rows.filter(row =>
-      Object.values(row).some(val => String(val).toLowerCase().includes(lower))
-    );
-  }, [data.rows, searchTerm]);
-
-  const sorted = useMemo(() => {
-    if (!sortCol || !sortDir) return filtered;
-    return [...filtered].sort((a, b) => {
-      const va = a[sortCol];
-      const vb = b[sortCol];
-      if (va == null && vb == null) return 0;
-      if (va == null) return 1;
-      if (vb == null) return -1;
-      if (typeof va === 'number' && typeof vb === 'number') {
-        return sortDir === 'asc' ? va - vb : vb - va;
-      }
-      const sa = String(va);
-      const sb = String(vb);
-      return sortDir === 'asc' ? sa.localeCompare(sb) : sb.localeCompare(sa);
-    });
-  }, [filtered, sortCol, sortDir]);
-
-  const paged = sorted.slice(page * pageSize, (page + 1) * pageSize);
-  const totalPages = Math.ceil(sorted.length / pageSize);
-
-  const exportToCSV = () => {
-    const headers = data.columns.join(',');
-    const rows = sorted.map(row =>
-      data.columns.map(col => {
-        const val = row[col] == null ? '' : String(row[col]);
-        return `"${val.replace(/"/g, '""')}"`;
-      }).join(',')
-    );
-    const csvContent = [headers, ...rows].join('\n');
-    // Use a Blob URL, not a data: URI — browsers cap data: URIs at a few MB,
-    // which silently blocks large exports. A leading BOM keeps Excel in UTF-8.
-    const blob = new Blob(['\ufeff' + csvContent], { type: 'text/csv;charset=utf-8;' });
-    const url = URL.createObjectURL(blob);
-    const link = document.createElement('a');
-    link.href = url;
-    link.download = `uap_data_export_${new Date().getTime()}.csv`;
-    document.body.appendChild(link);
-    link.click();
-    document.body.removeChild(link);
-    URL.revokeObjectURL(url);
-  };
-
-  const TableContent = (
-    <div className={`flex flex-col bg-surface ${isFullScreen ? 'h-full w-full fixed inset-0 z-[100] p-6 animate-in fade-in zoom-in duration-200' : 'relative'}`}>
-
-      {/* Table Header / Controls */}
-      <div className="flex items-center justify-between gap-4 mb-3">
-        <div className="flex items-center gap-2 flex-1 max-w-md relative">
-          <Search className="h-4 w-4 absolute left-3 text-text-muted" />
-          <input
-            type="text"
-            placeholder="Search current data..."
-            value={searchTerm}
-            onChange={(e) => { setSearchTerm(e.target.value); setPage(0); }}
-            className="w-full bg-elevated border border-border rounded-md pl-9 pr-4 py-1.5 text-xs focus:ring-1 focus:ring-accent outline-none"
-          />
-        </div>
-
-        <div className="flex items-center gap-2">
-          <button
-            onClick={exportToCSV}
-            title="Export to CSV"
-            className="flex items-center gap-2 px-3 py-1.5 rounded-md bg-elevated border border-border text-xs text-text-secondary hover:bg-accent hover:text-white transition-all shadow-sm"
-          >
-            <Download className="h-3.5 w-3.5" />
-            Export
-          </button>
-          <button
-            onClick={() => setIsFullScreen(!isFullScreen)}
-            title={isFullScreen ? "Exit Full Screen" : "Full Screen"}
-            className="p-1.5 rounded-md bg-elevated border border-border text-text-secondary hover:bg-purple hover:text-white transition-all shadow-sm"
-          >
-            {isFullScreen ? <Minimize2 className="h-4 w-4" /> : <Maximize2 className="h-4 w-4" />}
-          </button>
-          {isFullScreen && (
-            <button
-              onClick={() => setIsFullScreen(false)}
-              className="p-1.5 rounded-md bg-danger/10 text-danger hover:bg-danger hover:text-white transition-all"
-            >
-              <X className="h-4 w-4" />
-            </button>
-          )}
-        </div>
-      </div>
-
-      <div className="flex-1 overflow-auto border border-border rounded-lg bg-deep/50 shadow-inner" style={{ maxHeight: isFullScreen ? 'calc(100vh - 180px)' : maxHeight }}>
-        <table className="w-full border-collapse text-xs">
-          <thead className="sticky top-0 z-10 bg-deep border-b border-border shadow-sm">
-            <tr>
-              <th className="px-3 py-2.5 text-left text-[10px] font-bold uppercase tracking-wider text-text-muted bg-deep">
-                #
-              </th>
-              <th className="px-3 py-2.5 text-left text-[10px] font-bold uppercase tracking-wider text-text-muted bg-deep">
-                Actions
-              </th>
-              {data.columns.map((col) => (
-                <th
-                  key={col}
-                  onClick={() => handleSort(col)}
-                  className="cursor-pointer select-none px-3 py-2.5 text-left text-[10px] font-bold uppercase tracking-wider text-text-muted hover:text-accent transition-colors bg-deep"
-                >
-                  <div className="flex items-center gap-1.5">
-                    <span className="truncate">{col}</span>
-                    {sortCol === col ? (
-                      sortDir === 'asc' ? (
-                        <ChevronUp className="h-3 w-3 text-accent" />
-                      ) : (
-                        <ChevronDown className="h-3 w-3 text-accent" />
-                      )
-                    ) : (
-                      <ChevronsUpDown className="h-3 w-3 opacity-20" />
-                    )}
-                  </div>
-                </th>
-              ))}
-            </tr>
-          </thead>
-          <tbody className="divide-y divide-border/20">
-            {paged.map((row, idx) => (
-              <tr
-                key={idx}
-                className="group border-b border-border/30 transition-colors hover:bg-accent/5"
-              >
-                <td className="px-3 py-2 text-text-muted/60 font-mono text-[10px]">{page * pageSize + idx + 1}</td>
-                <td className="px-3 py-2 text-center">
-                  <button
-                    onClick={() => setSelectedRow(row)}
-                    className="p-1 rounded bg-elevated border border-border text-text-muted hover:text-purple hover:border-purple/50 transition-all opacity-0 group-hover:opacity-100"
-                    title="View Details"
-                  >
-                    <Eye className="h-3.5 w-3.5" />
-                  </button>
-                </td>
-                {data.columns.map((col) => {
-                  const val = row[col] != null ? String(row[col]) : '';
-                  return (
-                    <td
-                      key={col}
-                      className="max-w-48 truncate px-3 py-2 text-text-secondary font-medium tracking-tight"
-                      title={val.length > 30 ? val : ''}
-                    >
-                      {val}
-                    </td>
-                  );
-                })}
-              </tr>
-            ))}
-          </tbody>
-        </table>
-
-        {paged.length === 0 && (
-          <div className="py-20 text-center flex flex-col items-center gap-3">
-            <div className="p-4 rounded-full bg-elevated border border-border">
-              <Search className="h-8 w-8 text-text-muted" />
-            </div>
-            <div>
-              <p className="text-text-primary font-medium">No results found</p>
-              <p className="text-xs text-text-muted">Try adjusting your search terms</p>
-            </div>
-          </div>
-        )}
-      </div>
-
-      {/* Pagination */}
-      <div className="flex items-center justify-between border-t border-border mt-3 pt-3">
-        <div className="flex items-center gap-4">
-          <span className="text-[11px] text-text-muted font-medium bg-elevated px-2 py-1 rounded border border-border/50">
-            PAGE {page + 1} OF {totalPages || 1}
-          </span>
-          <span className="text-[11px] text-text-muted">
-            Showing <span className="text-text-primary">{page * pageSize + 1}–{Math.min((page + 1) * pageSize, sorted.length)}</span> of{' '}
-            <span className="text-text-primary font-bold">{sorted.length}</span> rows
-          </span>
-        </div>
-        <div className="flex items-center gap-2">
-          <button
-            disabled={page === 0}
-            onClick={() => setPage(page - 1)}
-            className="flex items-center gap-1 rounded-md border border-border px-3 py-1.5 text-xs font-semibold text-text-secondary bg-surface hover:bg-elevated transition-colors disabled:opacity-30 disabled:hover:bg-surface"
-          >
-            Previous
-          </button>
-          <button
-            disabled={page >= totalPages - 1}
-            onClick={() => setPage(page + 1)}
-            className="flex items-center gap-1 rounded-md border border-border px-3 py-1.5 text-xs font-semibold text-text-secondary bg-surface hover:bg-elevated transition-colors disabled:opacity-30 disabled:hover:bg-surface"
-          >
-            Next
-          </button>
-        </div>
-      </div>
-
-      {/* Row Detail Modal */}
-      {selectedRow && (
-        <div className="fixed inset-0 z-[110] flex items-center justify-center p-4 bg-background/80 backdrop-blur-sm animate-in fade-in duration-200">
-          <div className="w-full max-w-2xl bg-surface border border-border rounded-xl shadow-2xl overflow-hidden animate-in zoom-in-95 duration-200">
-            <div className="flex items-center justify-between px-6 py-4 border-b border-border bg-deep/50">
-              <div className="flex items-center gap-3">
-                <div className="p-2 rounded-lg bg-purple/20 text-purple">
-                  <FileText className="h-5 w-5" />
-                </div>
-                <h3 className="font-bold text-text-primary">Row Details</h3>
-              </div>
-              <button
-                onClick={() => setSelectedRow(null)}
-                className="p-1.5 rounded-full hover:bg-elevated text-text-muted hover:text-text-primary transition-colors"
-              >
-                <X className="h-5 w-5" />
-              </button>
-            </div>
-            <div className="max-h-[70vh] overflow-auto p-6">
-              <div className="grid grid-cols-1 gap-x-6 gap-y-4 sm:grid-cols-2">
-                {data.columns.map(col => (
-                  <div key={col} className="space-y-1 group">
-                    <label className="text-[10px] font-bold uppercase tracking-widest text-text-muted group-hover:text-accent transition-colors">
-                      {col}
-                    </label>
-                    <div className="p-3 bg-elevated rounded-lg border border-border/50 text-sm text-text-secondary break-words font-medium">
-                      {selectedRow[col] != null ? String(selectedRow[col]) : <span className="italic opacity-30">null</span>}
-                    </div>
-                  </div>
-                ))}
-              </div>
-            </div>
-            <div className="px-6 py-4 bg-deep/30 border-t border-border flex justify-end">
-              <button
-                onClick={() => setSelectedRow(null)}
-                className="px-6 py-2 rounded-lg bg-accent text-white font-bold text-sm hover:scale-[1.02] transform transition-all shadow-lg active:scale-95"
-              >
-                Close
-              </button>
-            </div>
-          </div>
-        </div>
-      )}
-    </div>
-  );
-
-  return TableContent;
-}

frontend/src/components/data/FilterPanel.tsx

@@ -1,310 +0,0 @@
-import { useState, useEffect, useRef } from 'react';
-import { Filter, X, Plus } from 'lucide-react';
-import { api } from '../../api/client';
-import type { ColumnStat } from '../../types';
-
-interface ActiveFilter {
-  id: number;
-  column: string;
-  type: string;
-  values?: string[];
-  min_val?: number;
-  max_val?: number;
-  pattern?: string;
-}
-
-interface FilterPanelProps {
-  columns: ColumnStat[];
-  onApply: (filters: ActiveFilter[]) => void;
-}
-
-type ValueCount = { value: string; count: number };
-
-let filterId = 0;
-
-/**
- * Searchable value picker for categorical filters.
- *
- * Shows the column's precomputed top values (from client-side column stats)
- * immediately, so values are always visible the moment the field is clicked.
- * Typing queries the server for the full set of matching values; if the
- * server is unavailable the picker falls back to filtering the local list.
- */
-function CategoricalValuePicker({
-  column,
-  selected,
-  onChange,
-  fallbackValues,
-}: {
-  column: string;
-  selected: string[];
-  onChange: (values: string[]) => void;
-  fallbackValues: ValueCount[];
-}) {
-  const [query, setQuery] = useState('');
-  const [results, setResults] = useState<ValueCount[]>(fallbackValues);
-  const [totalMatches, setTotalMatches] = useState(fallbackValues.length);
-  const [loading, setLoading] = useState(false);
-  const [serverBacked, setServerBacked] = useState(true);
-  const [open, setOpen] = useState(false);
-  const ref = useRef<HTMLDivElement>(null);
-
-  const localFilter = (q: string): ValueCount[] => {
-    const t = q.trim().toLowerCase();
-    return t ? fallbackValues.filter((v) => v.value.toLowerCase().includes(t)) : fallbackValues;
-  };
-
-  // Reset to the new column's local values when the column changes.
-  useEffect(() => {
-    setQuery('');
-    setResults(fallbackValues);
-    setTotalMatches(fallbackValues.length);
-    setServerBacked(true);
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [column]);
-
-  // Debounced server search; falls back to the local list on failure.
-  useEffect(() => {
-    let cancelled = false;
-    setLoading(true);
-    const t = setTimeout(() => {
-      api
-        .getColumnValues(column, query, 50)
-        .then((res) => {
-          if (cancelled) return;
-          setServerBacked(true);
-          setResults(res.values);
-          setTotalMatches(res.total_matches);
-        })
-        .catch(() => {
-          if (cancelled) return;
-          // Server (or its session data) unavailable — use the local list.
-          setServerBacked(false);
-          const local = localFilter(query);
-          setResults(local);
-          setTotalMatches(local.length);
-        })
-        .finally(() => {
-          if (!cancelled) setLoading(false);
-        });
-    }, 250);
-    return () => {
-      cancelled = true;
-      clearTimeout(t);
-    };
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [column, query]);
-
-  // Close the dropdown when clicking outside.
-  useEffect(() => {
-    if (!open) return;
-    const handler = (e: MouseEvent) => {
-      if (ref.current && !ref.current.contains(e.target as Node)) setOpen(false);
-    };
-    document.addEventListener('mousedown', handler);
-    return () => document.removeEventListener('mousedown', handler);
-  }, [open]);
-
-  const toggle = (value: string) => {
-    if (selected.includes(value)) onChange(selected.filter((v) => v !== value));
-    else onChange([...selected, value]);
-  };
-
-  return (
-    <div ref={ref} className="relative min-w-0 flex-1">
-      {selected.length > 0 && (
-        <div className="mb-1 flex flex-wrap gap-1">
-          {selected.map((v) => (
-            <span
-              key={v}
-              className="flex items-center gap-1 rounded bg-accent-dim px-1.5 py-0.5 text-[10px] text-white"
-            >
-              <span className="max-w-[140px] truncate" title={v}>
-                {v}
-              </span>
-              <button onClick={() => toggle(v)} className="hover:text-danger">
-                <X className="h-2.5 w-2.5" />
-              </button>
-            </span>
-          ))}
-        </div>
-      )}
-
-      <input
-        type="text"
-        placeholder={selected.length ? 'Add another value...' : 'Search values...'}
-        value={query}
-        onChange={(e) => setQuery(e.target.value)}
-        onFocus={() => setOpen(true)}
-        className="w-full rounded border border-border bg-deep px-2 py-1 text-xs text-text-primary placeholder:text-text-muted"
-      />
-
-      {open && (
-        <div className="absolute left-0 right-0 top-full z-20 mt-1 max-h-48 overflow-auto rounded border border-border bg-deep shadow-xl">
-          {results.length === 0 && loading && (
-            <div className="px-2 py-1.5 text-[10px] text-text-muted">Searching...</div>
-          )}
-          {results.length === 0 && !loading && (
-            <div className="px-2 py-1.5 text-[10px] text-text-muted">No matching values</div>
-          )}
-          {results.map((r) => {
-            const isSel = selected.includes(r.value);
-            return (
-              <button
-                key={r.value}
-                onClick={() => toggle(r.value)}
-                className={`flex w-full items-center justify-between gap-2 px-2 py-1 text-left text-[11px] hover:bg-elevated ${
-                  isSel ? 'text-accent' : 'text-text-secondary'
-                }`}
-              >
-                <span className="truncate" title={r.value}>
-                  {isSel ? '✓ ' : ''}
-                  {r.value}
-                </span>
-                <span className="shrink-0 text-text-muted">{r.count}</span>
-              </button>
-            );
-          })}
-          {totalMatches > results.length && (
-            <div className="px-2 py-1.5 text-[10px] text-text-muted">
-              +{totalMatches - results.length} more — refine your search
-            </div>
-          )}
-          {!serverBacked && (
-            <div className="border-t border-border px-2 py-1.5 text-[10px] text-amber-400/80">
-              Showing cached top values — reload the dataset for full search.
-            </div>
-          )}
-        </div>
-      )}
-    </div>
-  );
-}
-
-export function FilterPanel({ columns, onApply }: FilterPanelProps) {
-  const [filters, setFilters] = useState<ActiveFilter[]>([]);
-  const [open, setOpen] = useState(false);
-
-  const addFilter = () => {
-    if (columns.length === 0) return;
-    const col = columns[0];
-    const type = col.top_values ? 'categorical' : col.min != null ? 'numeric' : 'text';
-    setFilters([...filters, { id: ++filterId, column: col.name, type }]);
-  };
-
-  const removeFilter = (id: number) => {
-    const next = filters.filter((f) => f.id !== id);
-    setFilters(next);
-    onApply(next);
-  };
-
-  const updateFilter = (id: number, patch: Partial<ActiveFilter>) => {
-    setFilters((prev) => prev.map((f) => (f.id === id ? { ...f, ...patch } : f)));
-  };
-
-  const handleApply = () => {
-    onApply(filters);
-  };
-
-  return (
-    <div className="rounded-lg border border-border bg-surface">
-      <button
-        onClick={() => setOpen(!open)}
-        className="flex w-full items-center gap-2 px-4 py-2.5 text-sm text-text-secondary hover:text-text-primary"
-      >
-        <Filter className="h-4 w-4" />
-        Filters {filters.length > 0 && `(${filters.length})`}
-      </button>
-
-      {open && (
-        <div className="border-t border-border px-4 py-3">
-          <div className="space-y-2">
-            {filters.map((f) => {
-              const colInfo = columns.find((c) => c.name === f.column);
-              return (
-                <div key={f.id} className="flex items-start gap-2 rounded bg-raised p-2">
-                  <select
-                    value={f.column}
-                    onChange={(e) => {
-                      const newCol = columns.find((c) => c.name === e.target.value);
-                      const type = newCol?.top_values ? 'categorical' : newCol?.min != null ? 'numeric' : 'text';
-                      updateFilter(f.id, { column: e.target.value, type, values: undefined, pattern: undefined, min_val: undefined, max_val: undefined });
-                    }}
-                    className="mt-0.5 rounded border border-border bg-deep px-2 py-1 text-xs text-text-primary"
-                  >
-                    {columns.map((c) => (
-                      <option key={c.name} value={c.name}>
-                        {c.name}
-                      </option>
-                    ))}
-                  </select>
-
-                  {f.type === 'text' && (
-                    <input
-                      type="text"
-                      placeholder="Search pattern..."
-                      value={f.pattern ?? ''}
-                      onChange={(e) => updateFilter(f.id, { pattern: e.target.value })}
-                      className="mt-0.5 flex-1 rounded border border-border bg-deep px-2 py-1 text-xs text-text-primary placeholder:text-text-muted"
-                    />
-                  )}
-
-                  {f.type === 'numeric' && (
-                    <div className="mt-0.5 flex items-center gap-1">
-                      <input
-                        type="number"
-                        placeholder="Min"
-                        value={f.min_val ?? ''}
-                        onChange={(e) => updateFilter(f.id, { min_val: e.target.value ? Number(e.target.value) : undefined })}
-                        className="w-20 rounded border border-border bg-deep px-2 py-1 text-xs text-text-primary"
-                      />
-                      <span className="text-text-muted">-</span>
-                      <input
-                        type="number"
-                        placeholder="Max"
-                        value={f.max_val ?? ''}
-                        onChange={(e) => updateFilter(f.id, { max_val: e.target.value ? Number(e.target.value) : undefined })}
-                        className="w-20 rounded border border-border bg-deep px-2 py-1 text-xs text-text-primary"
-                      />
-                    </div>
-                  )}
-
-                  {f.type === 'categorical' && (
-                    <CategoricalValuePicker
-                      column={f.column}
-                      selected={f.values ?? []}
-                      onChange={(values) => updateFilter(f.id, { values })}
-                      fallbackValues={colInfo?.top_values ?? []}
-                    />
-                  )}
-
-                  <button
-                    onClick={() => removeFilter(f.id)}
-                    className="mt-1.5 text-text-muted hover:text-danger"
-                  >
-                    <X className="h-3.5 w-3.5" />
-                  </button>
-                </div>
-              );
-            })}
-          </div>
-
-          <div className="mt-3 flex gap-2">
-            <button
-              onClick={addFilter}
-              className="flex items-center gap-1.5 rounded border border-border px-3 py-1.5 text-xs text-text-secondary hover:bg-elevated"
-            >
-              <Plus className="h-3 w-3" /> Add Filter
-            </button>
-            <button
-              onClick={handleApply}
-              className="rounded bg-accent-dim px-3 py-1.5 text-xs font-medium text-white hover:bg-accent"
-            >
-              Apply Filters
-            </button>
-          </div>
-        </div>
-      )}
-    </div>
-  );
-}