diff --git a/.gitignore b/.gitignore index 2f04462a88318c408acebfe7a4c1c3b6873296a9..7b4d5398d85b5dff8885816f7ffe6fb331190354 100644 --- a/.gitignore +++ b/.gitignore @@ -1,191 +1 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so may need to be excluded from sync. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or binary, you wish to ignore these files, but for an app, you might want to check them in. -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or even -# fail to install them. -# Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -# pdm.lock - -# PEP 582; used by e.g. github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -.idea/ - -# Streamlit -.streamlit/ - -# VS Code -.vscode/ - -# Node.js -node_modules/ -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.npm/ - -# Dataset Files (Usually too large for git) -*.h5 -*.h5.1 -*.csv -*.jpeg -*.webp -*.pkl - -# Dist -frontend/dist/ -frontend2/dist/ - -# Large source PDFs (kept local, too big for git / HF Space) -UAP_PDFs/ - -# Document Preprocessing working dir (PDF corpus + intermediates) — data, not code. -# The pipeline *scripts* live in pipeline/ and ARE tracked; the data tree is not. -pipeline_data/ - -# Windows "downloaded from internet" metadata sidecar files -*Zone.Identifier - -# Superseded frontend backup + large generated cluster-viz HTML (kept local, -# excluded from the HF Space deploy to stay within its storage limit). -frontend_backup/ -frontend/uap_clusters_llm.html - -# Scratch notebooks -Untitled.ipynb +.streamlit diff --git a/.streamlit/config.toml b/.streamlit/config.toml new file mode 100644 index 0000000000000000000000000000000000000000..8cd4a3e942e0b5ec85385ff1c335c44d5df92ad1 --- /dev/null +++ b/.streamlit/config.toml @@ -0,0 +1,40 @@ +[browser] +pageTitle = "UAP ANALYTICS" +pageIcon = ":alien:" +layout = "wide" + +[server] +enableXsrfProtection = false +maxUploadSize=5000 +maxMessageSize=5000 + + +[theme] +# Primary accent for interactive elements +primaryColor = "#FFA500" + +# Background color for the main content area +#backgroundColor = "#273346" + +# Background color for sidebar and most interactive widgets +#secondaryBackgroundColor = "#B9F1C0" + +# Color used for almost all text +#textColor = "#FFFFFF" + +# Font family for all text in the app, except code blocks +# Accepted values (serif | sans serif | monospace) +# Default: "sans serif" +font = "sans serif" + +# Base theme (light or dark) +base = "dark" + + +[runner] +magicEnabled = true +fastReruns = false + + +[client] +toolbarMode = "auto" diff --git a/.streamlit/credentials.toml b/.streamlit/credentials.toml new file mode 100644 index 0000000000000000000000000000000000000000..0737dcd756948b110d5102097c910396799f133a --- /dev/null +++ b/.streamlit/credentials.toml @@ -0,0 +1,2 @@ +[general] +email = "" diff --git a/.streamlit/secrets.toml b/.streamlit/secrets.toml new file mode 100644 index 0000000000000000000000000000000000000000..b5f19a4815a246417bf59ca23be98dd815c6b3e3 --- /dev/null +++ b/.streamlit/secrets.toml @@ -0,0 +1,14 @@ +testing_mode = false +payment_provider = "stripe" +stripe_api_key_test = 'rk_test_51PYs5M2LWM8BdEzK3ggKUMfcoVpwpmWABeiewJZp797aMvxWzlmaDe70svq5wajxiun4x98OhQZQB1lqP3AsQfDS009pbiZHSx' +#stripe_api_key = 'pk_live_51PYs5M2LWM8BdEzK2lhKscFJJ8l8Z0CxY4xiLUTOoRjQnri1Qcf47Fhf3SAXH9P8jyPKYxfo3xEpHpHD5n8jMbqE00E3gRdxPF' +stripe_api_key = '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' +stripe_link_test = 'https://buy.stripe.com/test_4gw0390KX0ojc8w7ss' +stripe_link = 'https://buy.stripe.com/bIYcP31PZ4Jwdgs288' +client_id = '628411883365-dfkuut4shontl77uge7mta9514hambkc.apps.googleusercontent.com' +client_secret = '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' +redirect_url_test = 'https://huggingface.co/spaces/Ashoka74/UFOSINT' +redirect_url = 'https://huggingface.co/spaces/UFOSINT/UAP-Data-Analysis-Tool/' +GEMINI_KEY = 'AIzaSyAEALAXiaE1HcD8qcN1duY4OtmUDfYqquk' +COHERE_KEY = 'UOGoge1RICTXAb710UrE0QQSftv6qZx8ysJKXY6j' +OPENAI_KEY = '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' \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index f05991563fdb74c969e1aa7f309f6070cbdc72d5..0000000000000000000000000000000000000000 --- a/CLAUDE.md +++ /dev/null @@ -1,115 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -UAP-Data-Analysis-Tool (also known as UFOSINT) is a Streamlit-based data analysis application for UFO/UAP (Unidentified Aerial Phenomena) sighting reports. It provides text parsing, clustering, statistical analysis, geospatial visualization, and magnetic anomaly correlation capabilities. - -## Commands - -### Running the Application - -```bash -# Streamlit app (main interface) -uv run streamlit run app.py - -# Gradio app (alternative interface) -uv run gradio gradio_app.py -``` - -### Dependency Management - -```bash -# Install dependencies using uv (Python 3.12+) -uv sync - -# Or using pip -pip install -r requirements.txt -``` - -### Testing/Running Individual Pages - -Each Streamlit page can be tested individually: -```bash -uv run streamlit run parsing.py -uv run streamlit run analyzing.py -uv run streamlit run rag_search.py -uv run streamlit run magnetic.py -uv run streamlit run map.py -``` - -## Architecture - -### Core Analysis Pipeline - -The application follows a multi-step NLP analysis pipeline: - -1. **Parsing (`parsing.py`)** - Uses OpenAI GPT-4o-mini to extract structured JSON features from raw UAP report text. The JSON schema is defined in `config.py` (`FORMAT_LONG`). - -2. **Analysis (`analyzing.py`)** - Performs dimensionality reduction (UMAP) and clustering (HDBSCAN) on text embeddings, then trains XGBoost classifiers to identify feature correlations. - -3. **RAG Search (`rag_search.py`)** - Implements semantic search using Cohere's rerank API to find relevant reports based on natural language queries. - -4. **Magnetic Analysis (`magnetic.py`)** - Correlates UAP sightings with InterMagnet station data using Dynamic Time Warping (FastDTW). - -5. **Map Visualization (`map.py`)** - Interactive Kepler.gl maps showing sighting locations with proximity analysis to military bases and nuclear facilities. - -### Key Classes (`uap_analyzer.py`) - -- **`UAPParser`** - Concurrent OpenAI API calls for JSON extraction with exponential backoff -- **`UAPAnalyzer`** - Text embedding (sentence-transformers/e5-large-v2), UMAP reduction, HDBSCAN clustering, TF-IDF cluster naming, cluster merging via cosine similarity -- **`UAPVisualizer`** - XGBoost prediction plots, confusion matrices, Cramer's V heatmaps, treemaps - -### Utils Module (`utils/`) - -Enhanced utilities providing: -- `DataProcessor` - DataFrame filtering with Streamlit UI -- `UAP_Visualizer` - Interactive Plotly visualizations -- `SessionStateManager` - Streamlit session state handling -- `EmbeddingCacheManager` - Caches sentence-transformer embeddings to avoid recomputation -- `MemoryManager` - GPU memory management for CUDA operations -- `UAP_Pipeline` - End-to-end analysis pipeline orchestration - -### Data Flow - -``` -Raw Text Reports (CSV/XLSX) - └── parsing.py (GPT-4o-mini) → Structured JSON - └── analyzing.py → Embeddings → UMAP → HDBSCAN clusters - └── XGBoost predictions + Cramer's V correlations - └── map.py (Kepler.gl visualization) -``` - -### Session State Keys - -The app uses Streamlit session state extensively. Key variables: -- `parsed_responses` / `parsed_responses_df` - Parsed JSON data -- `analyzers` / `clusters` / `col_names` - Analysis results -- `stage` - UI workflow state -- `api_key_valid` - OpenAI key validation status - -## Configuration - -### API Keys - -Secrets are stored in `.streamlit/secrets.toml` (not committed). Required keys: -- `OPENAI_KEY` - For GPT-4o-mini parsing -- `GEMINI_KEY` - For Gemini Pro summarization -- `COHERE_KEY` - For rerank search - -### Data Files - -- `final_ufoseti_dataset.h5` - Pre-parsed UAP dataset -- `parsed_files_distance_embeds.h5` - Dataset with embeddings -- `global_power_plant_database.csv` - Nuclear facility locations -- `secret_bases.csv` - Military base locations -- `*.kgl` - Kepler.gl map configuration files - -## GPU Requirements - -The embedding model (`embaas/sentence-transformers-e5-large-v2`) and XGBoost run on CUDA by default. The code includes `torch.cuda.empty_cache()` calls for memory management. - -## HuggingFace Deployment - -The app is configured for HuggingFace Spaces deployment (see `README.md` metadata). The `sdk_version: 1.36.0` specifies the Streamlit version. diff --git a/ENHANCED_PIPELINE_README.md b/ENHANCED_PIPELINE_README.md deleted file mode 100644 index 4ec4092fcd2dc86c716c482e33664f11f63e86d7..0000000000000000000000000000000000000000 --- a/ENHANCED_PIPELINE_README.md +++ /dev/null @@ -1,287 +0,0 @@ -# Enhanced Dynamic Filtering and Visualization Pipeline - -## Overview - -This document describes the major improvements made to the UAP Data Analysis Tool's dynamic filtering and visualization pipeline. The enhancements focus on performance, usability, and advanced interactive features. - -## 🚀 Key Improvements - -### 1. Enhanced Data Processing (`utils/data_processing.py`) - -#### Intelligent Data Profiling -- **Automatic column type detection** with statistical analysis -- **Memory usage optimization** and performance monitoring -- **Smart categorization** of columns (categorical, numeric, datetime, text) -- **High cardinality column detection** for special handling - -#### Advanced Filtering System -- **Quick Filter Presets**: Pre-configured filters for common scenarios - - Remove outliers using IQR method - - Top categories only - - Remove rare categories (< 1% frequency) - - Recent data filtering -- **Multi-modal Filtering**: - - Range, percentile, and standard deviation filters for numeric data - - Smart selection, top-N, and exclusion modes for categorical data - - Advanced date filtering with relative periods - - Text filtering with regex support and length-based filtering - -#### Performance Optimizations -- **Intelligent caching** with dataframe hashing -- **Performance monitoring** decorators -- **Parallel processing** for data parsing operations -- **Memory-efficient operations** with smart sampling - -### 2. Enhanced Visualization System (`utils/visualization.py`) - -#### Interactive Visualizations -- **Plotly-based interactive charts** replacing static matplotlib plots -- **Smart sampling** for large datasets maintaining data distribution -- **Progressive rendering** to handle datasets of any size -- **Drill-down capabilities** in treemaps and other charts - -#### New Visualization Types -1. **Interactive Scatter Plots** - - Color and size encoding - - Intelligent sampling for performance - - Hover data with context - - Sampling indicators for transparency - -2. **Enhanced Histograms** - - Box plots integration - - Interactive binning - - Statistical overlays - -3. **Interactive Treemaps** - - Drill-down capabilities - - Percentage and count displays - - Color-coded hierarchies - -4. **Correlation Matrices** - - Interactive heatmaps - - Multiple correlation methods (Pearson, Spearman, Kendall) - - Hover tooltips with exact values - -5. **Time Series Plots** - - Range selectors and sliders - - Multiple series support - - Resampling options - - Zoom and pan capabilities - -6. **Dashboard Layouts** - - Multi-chart dashboards - - Configurable layouts (2x2, vertical) - - Synchronized interactions - -#### Performance Features -- **Smart sampling algorithms** preserving data distribution -- **Stratified sampling** for categorical data -- **Caching at multiple levels** (Streamlit cache + custom cache) -- **Memory-aware rendering** with automatic optimization - -### 3. Session State Management (`utils/session_manager.py`) - -#### Enhanced State Handling -- **Centralized session state management** -- **Visualization caching** for improved performance -- **Filter state persistence** across app interactions -- **Memory management** for large datasets - -### 4. Application Integration - -#### Updated Applications -All main applications now use the enhanced pipeline: - -1. **Analyzing App** (`analyzing.py`) - - Enhanced filtering with quick presets - - Interactive visualization tabs - - Performance monitoring - - Re-analysis on filtered data - -2. **Map App** (`map.py`) - - Map-optimized filtering (datetime to string conversion) - - Geographic data handling improvements - - Enhanced coordinate detection - -3. **Other Apps** can be easily updated using the same pattern - -## 📊 Usage Examples - -### Basic Enhanced Filtering -```python -from utils.data_processing import DataProcessor - -# Enhanced filtering with all features -filtered_df = DataProcessor.filter_dataframe_enhanced( - df, - enable_quick_filters=False, - enable_advanced_filters=True -) -``` - -### Interactive Visualizations -```python -from utils.visualization import UAP_Visualizer - -# Interactive scatter plot with smart sampling -fig = UAP_Visualizer.plot_interactive_scatter( - df, 'latitude', 'longitude', - color_col='shape', - max_points=10000 -) -st.plotly_chart(fig, use_container_width=True) - -# Correlation matrix -fig = UAP_Visualizer.plot_correlation_matrix(df[numeric_columns]) -st.plotly_chart(fig, use_container_width=True) -``` - -### Data Profiling -```python -# Get intelligent data profile -profile = DataProcessor.profile_data(df) -print(f"Categorical columns: {len(profile['categorical_columns'])}") -print(f"Numeric columns: {len(profile['numeric_columns'])}") -print(f"Memory usage: {profile['memory_usage'] / 1024**2:.1f} MB") -``` - -## 🎯 Performance Improvements - -### Before vs After -- **Filtering Speed**: 3-5x faster with intelligent caching -- **Visualization Rendering**: 2-10x faster with smart sampling -- **Memory Usage**: 30-50% reduction for large datasets -- **User Experience**: Instant feedback with progressive loading - -### Smart Sampling Benefits -- Maintains statistical properties of data -- Preserves category distributions -- Transparent to users (shows sampling info) -- Configurable sampling limits - -### Caching Strategy -- **Multi-level caching**: Streamlit + custom application cache -- **Intelligent cache keys**: Based on data hashes -- **Automatic cache invalidation**: When data changes -- **Memory-aware caching**: Prevents memory overflow - -## 🔧 Configuration Options - -### Performance Tuning -```python -# Adjust sampling limits -UAP_Visualizer.plot_interactive_scatter(df, x, y, max_points=5000) - -# Enable/disable performance mode -DataProcessor.filter_dataframe_enhanced( - df, - enable_quick_filters=False, # Quick preset filters - enable_advanced_filters=True # Full filtering interface -) -``` - -### Visualization Customization -```python -# Custom color schemes -fig = UAP_Visualizer.plot_correlation_matrix(df, method='spearman') - -# Time series with resampling -fig = UAP_Visualizer.plot_time_series( - df, 'date_column', ['value1', 'value2'], - resample_freq='M' # Monthly resampling -) -``` - -## 🚀 Getting Started - -### 1. Install Dependencies -The enhanced pipeline requires additional dependencies (already in requirements.txt): -- `plotly` - Interactive visualizations -- `pandas` - Enhanced data processing -- `streamlit` - Caching and UI components - -### 2. Update Existing Code -Replace old filter functions: -```python -# Old way -filtered_df = filter_dataframe(df) - -# New way -from utils.data_processing import DataProcessor -filtered_df = DataProcessor.filter_dataframe_enhanced(df) -``` - -### 3. Use Enhanced Visualizations -```python -# Replace matplotlib plots with interactive versions -from utils.visualization import UAP_Visualizer - -# Interactive histogram instead of static -fig = UAP_Visualizer.plot_interactive_histogram(df, 'column_name') -st.plotly_chart(fig, use_container_width=True) -``` - -### 4. Try the Demo -Run the enhanced example: -```bash -streamlit run utils/enhanced_example.py -``` - -## 🔮 Future Enhancements - -### Planned Features -1. **Real-time Filtering**: WebSocket-based live data filtering -2. **Advanced Analytics**: Statistical tests and ML model integration -3. **Export Capabilities**: Enhanced data export with filter preservation -4. **Custom Visualizations**: User-defined chart types -5. **Performance Profiling**: Built-in performance analytics dashboard - -### Extensibility -The new architecture is designed for easy extension: -- Add new filter types in `DataProcessor` -- Create custom visualizations in `UAP_Visualizer` -- Extend session management for new use cases - -## 📈 Impact - -### User Experience -- **Faster interactions**: Immediate feedback on all operations -- **Better insights**: Interactive visualizations reveal patterns -- **Easier exploration**: Quick filters and smart defaults -- **Transparent performance**: Users see sampling and processing info - -### Developer Experience -- **Cleaner code**: Centralized utilities eliminate duplication -- **Better maintainability**: Single source of truth for filtering/visualization -- **Performance monitoring**: Built-in performance tracking -- **Easy extension**: Modular architecture for new features - -### System Performance -- **Scalability**: Handles datasets from thousands to millions of rows -- **Memory efficiency**: Smart sampling and caching prevent memory issues -- **Response times**: Sub-second response for most operations -- **Resource usage**: Optimized CPU and memory utilization - -## 🛠️ Technical Details - -### Architecture -``` -UAP Data Analysis Tool -├── utils/ -│ ├── data_processing.py # Enhanced filtering and data ops -│ ├── visualization.py # Interactive visualizations -│ ├── session_manager.py # State management -│ └── enhanced_example.py # Complete demo -├── analyzing.py # Updated with enhanced features -├── map.py # Updated with enhanced features -└── other apps... # Can be updated similarly -``` - -### Key Classes -- `DataProcessor`: Centralized data operations with intelligent caching -- `UAP_Visualizer`: Interactive visualization factory with performance optimization -- `SessionStateManager`: Enhanced state management with visualization caching - -The enhanced pipeline represents a significant upgrade to the UAP Data Analysis Tool, providing better performance, richer interactivity, and a superior user experience while maintaining backward compatibility and extensibility for future enhancements. - - diff --git a/PLAN.md b/PLAN.md deleted file mode 100644 index ca2d33eb5d192545e7238ecf0841110374a08eef..0000000000000000000000000000000000000000 --- a/PLAN.md +++ /dev/null @@ -1,313 +0,0 @@ -# Implementation Plan: React + FastAPI Architecture - -## Overview - -Convert the Streamlit-based UAP Data Analysis Tool to a modern React frontend with FastAPI backend, maintaining all existing functionality. - -## Architecture - -``` -┌─────────────────────────────────────────────────────────────┐ -│ React Frontend │ -│ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌───────┐ │ -│ │ Parsing │ │Analysis │ │ Search │ │Magnetic │ │ Map │ │ -│ └────┬────┘ └────┬────┘ └────┬────┘ └────┬────┘ └───┬───┘ │ -└───────┼──────────┼──────────┼──────────┼──────────┼───────┘ - │ │ │ │ │ - ▼ ▼ ▼ ▼ ▼ -┌─────────────────────────────────────────────────────────────┐ -│ FastAPI Backend │ -│ /api/parse /api/analyze /api/search /api/magnetic /api/map │ -└─────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Existing Python Services │ -│ UAPParser UAPAnalyzer UAPVisualizer Cohere InterMagnet │ -└─────────────────────────────────────────────────────────────┘ -``` - -## File Structure - -``` -UAP-Data-Analysis-Tool/ -├── api/ # FastAPI backend -│ ├── __init__.py -│ ├── main.py # FastAPI app entry point -│ ├── config.py # Settings and API keys -│ ├── routes/ -│ │ ├── __init__.py -│ │ ├── upload.py # File upload endpoints -│ │ ├── parse.py # OpenAI parsing endpoints -│ │ ├── analyze.py # UMAP/HDBSCAN/XGBoost endpoints -│ │ ├── search.py # Cohere rerank endpoints -│ │ ├── magnetic.py # InterMagnet correlation endpoints -│ │ └── map.py # Geospatial data endpoints -│ ├── services/ -│ │ ├── __init__.py -│ │ ├── parser_service.py # Wraps UAPParser -│ │ ├── analyzer_service.py # Wraps UAPAnalyzer -│ │ ├── visualizer_service.py # Wraps UAPVisualizer -│ │ ├── search_service.py # Cohere rerank logic -│ │ ├── magnetic_service.py # InterMagnet API + DTW -│ │ └── map_service.py # Kepler.gl data prep -│ ├── models/ -│ │ ├── __init__.py -│ │ ├── schemas.py # Pydantic request/response models -│ │ └── jobs.py # Background job tracking -│ └── utils/ -│ ├── __init__.py -│ ├── file_handler.py # CSV/Excel/HDF5 handling -│ └── serialization.py # NumPy/DataFrame serialization -│ -├── frontend/ # React frontend -│ ├── package.json -│ ├── tailwind.config.js -│ ├── vite.config.js -│ ├── index.html -│ ├── src/ -│ │ ├── main.jsx -│ │ ├── App.jsx -│ │ ├── api/ -│ │ │ └── client.js # Axios/fetch wrapper -│ │ ├── components/ -│ │ │ ├── layout/ -│ │ │ │ ├── Navbar.jsx -│ │ │ │ ├── Sidebar.jsx -│ │ │ │ └── Layout.jsx -│ │ │ ├── common/ -│ │ │ │ ├── FileUpload.jsx -│ │ │ │ ├── DataTable.jsx -│ │ │ │ ├── LoadingSpinner.jsx -│ │ │ │ └── ErrorBoundary.jsx -│ │ │ ├── charts/ -│ │ │ │ ├── Treemap.jsx -│ │ │ │ ├── Histogram.jsx -│ │ │ │ ├── ScatterPlot.jsx -│ │ │ │ ├── Heatmap.jsx -│ │ │ │ └── ConfusionMatrix.jsx -│ │ │ └── map/ -│ │ │ └── KeplerMap.jsx -│ │ ├── pages/ -│ │ │ ├── Home.jsx -│ │ │ ├── Parsing.jsx -│ │ │ ├── Analysis.jsx -│ │ │ ├── Search.jsx -│ │ │ ├── Magnetic.jsx -│ │ │ └── Map.jsx -│ │ ├── hooks/ -│ │ │ ├── useFileUpload.js -│ │ │ ├── useAnalysis.js -│ │ │ └── useWebSocket.js -│ │ ├── store/ -│ │ │ └── index.js # Zustand or React Context -│ │ └── styles/ -│ │ └── globals.css -│ └── public/ -│ └── assets/ -``` - -## Implementation Steps - -### Phase 1: FastAPI Backend Setup - -#### Step 1.1: Create API structure and main entry point -- Create `api/` directory structure -- Set up FastAPI app with CORS middleware -- Configure settings from environment/secrets -- Add health check endpoint - -#### Step 1.2: Create Pydantic schemas -- Define request models for each endpoint -- Define response models with proper typing -- Create job status models for async operations - -#### Step 1.3: Implement file upload endpoint -- POST `/api/upload` - Accept CSV/Excel files -- Store uploaded files temporarily -- Return file ID and column list -- Support chunked uploads for large files - -#### Step 1.4: Implement parsing endpoints -- POST `/api/parse/start` - Start async parsing job -- GET `/api/parse/status/{job_id}` - Check job status -- GET `/api/parse/result/{job_id}` - Get parsed results -- Wrap existing UAPParser with proper error handling - -#### Step 1.5: Implement analysis endpoints -- POST `/api/analyze/start` - Start clustering analysis -- GET `/api/analyze/status/{job_id}` - Check status -- GET `/api/analyze/clusters/{job_id}` - Get cluster data -- GET `/api/analyze/embeddings/{job_id}` - Get 2D embeddings for visualization -- GET `/api/analyze/predictions/{job_id}` - Get XGBoost results - -#### Step 1.6: Implement search endpoints -- POST `/api/search/rerank` - Cohere rerank search -- Return ranked results with relevance scores - -#### Step 1.7: Implement magnetic endpoints -- GET `/api/magnetic/stations` - List InterMagnet stations -- POST `/api/magnetic/correlate` - Run DTW correlation -- Return correlation results and time series data - -#### Step 1.8: Implement map endpoints -- GET `/api/map/sightings` - Get sighting GeoJSON -- GET `/api/map/bases` - Get military bases data -- GET `/api/map/plants` - Get nuclear facilities data -- GET `/api/map/config` - Get Kepler.gl config - -### Phase 2: React Frontend Setup - -#### Step 2.1: Initialize React project -- Create Vite + React project in `frontend/` -- Install dependencies: react-router, axios, recharts/plotly, kepler.gl -- Configure Tailwind CSS -- Set up project structure - -#### Step 2.2: Create layout components -- Navbar with navigation links -- Sidebar for feature options -- Main layout wrapper -- Responsive design - -#### Step 2.3: Create common components -- FileUpload with drag-and-drop -- DataTable with sorting/filtering -- LoadingSpinner and progress indicators -- Error boundary and toast notifications - -#### Step 2.4: Create chart components -- Treemap using Plotly.js -- Histogram using Recharts -- ScatterPlot for embeddings visualization -- Heatmap for Cramer's V and confusion matrix -- Feature importance bar chart - -#### Step 2.5: Create Kepler.gl map component -- Integrate kepler.gl React component -- Handle data layers dynamically -- Support filtering by attributes - -### Phase 3: Feature Pages - -#### Step 3.1: Parsing page -- File upload interface -- Column selector -- Custom JSON schema editor (optional) -- Progress indicator for parsing -- Results table with download option - -#### Step 3.2: Analysis page -- Dataset loader (upload or use parsed data) -- Column multi-selector for analysis -- Visualization tabs: Embeddings, Clusters, Predictions, Correlations -- Interactive charts with tooltips - -#### Step 3.3: Search page -- Dataset display -- Query input -- Column selector for search scope -- Ranked results with relevance scores -- Click-to-expand details - -#### Step 3.4: Magnetic page -- Date range selector -- Location input (lat/lon or from dataset) -- Station selector -- Correlation results with time series chart - -#### Step 3.5: Map page -- Full-screen Kepler.gl map -- Layer toggles (sightings, bases, plants) -- Filter controls -- Export functionality - -### Phase 4: Integration and Polish - -#### Step 4.1: State management -- Set up Zustand store for global state -- Persist uploaded data across pages -- Handle authentication state (API keys) - -#### Step 4.2: WebSocket for long-running tasks -- Add WebSocket endpoint for job progress -- Real-time updates during parsing/analysis - -#### Step 4.3: Error handling -- Consistent error responses from API -- User-friendly error messages in frontend -- Retry logic for failed requests - -#### Step 4.4: Testing -- API endpoint tests with pytest -- Component tests with React Testing Library - -## Key API Endpoints Summary - -| Method | Endpoint | Description | -|--------|----------|-------------| -| POST | `/api/upload` | Upload CSV/Excel file | -| POST | `/api/parse/start` | Start parsing job | -| GET | `/api/parse/status/{job_id}` | Get parsing status | -| GET | `/api/parse/result/{job_id}` | Get parsed data | -| POST | `/api/analyze/start` | Start analysis job | -| GET | `/api/analyze/clusters/{job_id}` | Get cluster results | -| GET | `/api/analyze/embeddings/{job_id}` | Get 2D embeddings | -| POST | `/api/search/rerank` | Semantic search | -| GET | `/api/magnetic/stations` | List stations | -| POST | `/api/magnetic/correlate` | Run correlation | -| GET | `/api/map/sightings` | Get sighting GeoJSON | -| GET | `/api/map/bases` | Get bases GeoJSON | - -## Dependencies to Add - -### Backend (add to pyproject.toml) -``` -fastapi -uvicorn[standard] -python-multipart -aiofiles -websockets -``` - -### Frontend (package.json) -```json -{ - "dependencies": { - "react": "^18.2.0", - "react-dom": "^18.2.0", - "react-router-dom": "^6.x", - "axios": "^1.x", - "plotly.js": "^2.x", - "react-plotly.js": "^2.x", - "kepler.gl": "^3.x", - "react-dropzone": "^14.x", - "@tanstack/react-table": "^8.x", - "zustand": "^4.x", - "react-hot-toast": "^2.x" - }, - "devDependencies": { - "vite": "^5.x", - "tailwindcss": "^3.x", - "autoprefixer": "^10.x", - "postcss": "^8.x" - } -} -``` - -## Running the Application - -```bash -# Terminal 1: Start FastAPI backend -cd api && uvicorn main:app --reload --port 8000 - -# Terminal 2: Start React frontend -cd frontend && npm run dev -``` - -## Notes - -- Long-running tasks (parsing, analysis) use background jobs with polling or WebSocket updates -- Embeddings are stored server-side and referenced by job_id to avoid large payloads -- Visualizations are generated as Plotly JSON for interactive frontend rendering -- The existing `uap_analyzer.py` and `utils/` modules are reused as services diff --git a/README.md b/README.md index 5d7a6eb9675a14144f02c5db7cd850d783aa6dea..db15b7b72e1424b43b2773b9f36a2380d3fa4c1a 100644 --- a/README.md +++ b/README.md @@ -5,57 +5,10 @@ colorFrom: green colorTo: yellow sdk: streamlit sdk_version: 1.36.0 -python_version: "3.12" app_file: app.py pinned: false license: apache-2.0 short_description: UFO/UAP AI Analyst --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference - ---- - -## Modern FrontEnd (React + FastAPI) - -The repo ships a decoupled web stack alongside the Streamlit app: - -- **Backend** — FastAPI service under `./api` (routes in `api/routes/`, services in `api/services/`, models in `api/models/`). -- **Frontend** — Vite + React + TypeScript + Tailwind + Zustand under `./frontend`, talking to the backend through a Vite proxy. - -### Prerequisites -- Python 3.11+ with `fastapi` and `uvicorn` installed (covered by `requirements.txt` / `pyproject.toml`). -- Node 20+ for the frontend. - -### 1. Start the backend (FastAPI) - -From the repo root: - -```bash -uvicorn api.main:app --reload --port 8000 -``` - -The `--reload` flag picks up code changes automatically. - -### 2. Start the frontend (Vite + React) - -```bash -cd frontend -npm install # first time only -npm run dev -``` - -Vite starts on port **5173** and proxies every `/api/*` request to `http://localhost:8000` (see `frontend/vite.config.ts`), so the React app calls FastAPI transparently — no CORS configuration needed. - -### 3. Open the app - -http://localhost:5173 - -### Production build - -```bash -cd frontend -npm run build # output in frontend/dist -``` - -Serve the `dist/` artifacts behind any static host and keep `uvicorn api.main:app` running for the API. \ No newline at end of file +Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference \ No newline at end of file diff --git a/STREAMLIT_HANDOFF.md b/STREAMLIT_HANDOFF.md deleted file mode 100644 index 41da16be2c7e52c2bbbda5cde25fd8d1008fab4c..0000000000000000000000000000000000000000 --- a/STREAMLIT_HANDOFF.md +++ /dev/null @@ -1,545 +0,0 @@ -# Handoff: UAP Embeddings → Streamlit Semantic Search - -Everything you need to build a Streamlit app that does semantic search over the -UAP archive embeddings currently sitting in a Neon Postgres + pgvector database. - ---- - -## 1. Context in one paragraph - -A previous session embedded all of **UAP Release 2 (5/22/26)** — 49 DoD UAP -video clips and 7 NASA Apollo/Mercury audio recordings — into a Neon Postgres -database using **Google Gemini `gemini-embedding-2-preview`** (768-dim, cosine -similarity, indexed with HNSW). The pipeline lives in `embeddings_v2.py` at the -repo root. Your job is a Streamlit UI that lets users type a query (or upload an -image), embed it with the same model, and return ranked matches with playable -media. - ---- - -## 2. What's in the database right now - -``` -source_type rows distinct assets -video_chunk 154 49 DVIDS UAP video clips (Release 2) -pdf_page 126 5 source documents (DOW-D017 [116p], DOE-D002 [4p], - CIA-D001 [3p], DOE-D001 [2p], DOE-D003 [1p]) -audio_clip 27 7 NASA Apollo/Mercury audio recordings (Release 2) -TOTAL 307 61 assets all release='PURSUE_2' release_date=2026-05-22 -``` - -- All current rows use `user_id = '00000000-0000-0000-0000-000000000001'` (a - placeholder UUID — the schema is multi-tenant but this archive has one tenant). -- `parent_id` is `dvids_{asset_id}` for media rows (e.g. `dvids_1007706`); doc - slugs like `dow-uap-d017` for `pdf_page` rows. -- `source_id` is `{parent_id}:{start_ms}-{end_ms}` for media chunks and - `{parent_id}:p{NNNN}` for PDF pages (e.g. `dow-uap-d017:p0017`). -- Vector dimension is **768**. Queries must be 768-dim too. -- Every row carries the new `release` (`'PURSUE_2'`) and `release_date` - (`2026-05-22`) columns — filter on these in the UI when more releases land. -- One pending video (`1007708`, the 513 MB outlier) was not ingested; it can be - added later — not a blocker for the UI. -- Nothing from earlier releases (Release 1, NARA-CIA, FBI photos, etc.) is - embedded yet. If you build the UI to filter on `release` / `parent_id` - patterns or future source types, leave it open. - ---- - -## 3. Schema reference - -```sql -CREATE TABLE embeddings ( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - source_type TEXT NOT NULL, -- 'video_chunk' | 'audio_clip' | 'pdf_page' (more later) - source_id TEXT NOT NULL, -- '{parent_id}:{start_ms}-{end_ms}' for chunks; '{slug}:p{NNNN}' for pages - user_id UUID NOT NULL, - organization_id UUID, - embedding VECTOR(768) NOT NULL, - embedded_image_url TEXT, -- video/audio: DVIDS page URL; pdf_page: whole-PDF war.gov URL - embedded_text TEXT, -- caption used during embed (Title + Blurb; or metadata + OCR for pdf_page) - start_seconds REAL, -- chunk start (NULL for pdf_page) - end_seconds REAL, -- chunk end (NULL for pdf_page) - parent_id TEXT, -- 'dvids_1007706' for media; doc slug like 'dow-uap-d017' for pages - release TEXT NOT NULL DEFAULT 'PURSUE_2', -- campaign tag (filter on this in the UI) - release_date DATE NOT NULL DEFAULT '2026-05-22', -- when the source documents were publicly released - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - CONSTRAINT uq_embeddings_source UNIQUE (source_type, source_id) -); - --- Already created: -CREATE INDEX idx_embeddings_embedding ON embeddings USING hnsw (embedding vector_cosine_ops); -CREATE INDEX idx_embeddings_parent_id ON embeddings (parent_id) WHERE parent_id IS NOT NULL; -CREATE INDEX idx_embeddings_user_id ON embeddings (user_id); -``` - -Cosine search uses pgvector's `<=>` operator (distance, lower = closer). -Convert to similarity with `1 - (embedding <=> query)`. - ---- - -## 4. Secrets — required, not in this file - -Set as env vars (or Streamlit `secrets.toml`): - -```bash -DATABASE_URL = -GEMINI_API_KEY = -``` - -The Neon string must include `?sslmode=require`. Ask the user to paste the -values from their Neon dashboard and Google AI Studio — they're not embedded -here on purpose. The previous session ran against a Neon project owned by the -user, and the password / key from that session should be considered exposed -and rotated. - -**Streamlit secrets.toml** (recommended over raw env vars): - -```toml -# .streamlit/secrets.toml -- DO NOT COMMIT -DATABASE_URL = "postgresql://USER:PASSWORD@ep-xxxx.REGION.aws.neon.tech/neondb?sslmode=require" -GEMINI_API_KEY = "AIza..." -``` - -Read in app with `st.secrets["DATABASE_URL"]`. - ---- - -## 5. Dependencies - -```bash -pip install streamlit google-genai pillow requests "psycopg[binary]" pgvector -``` - -The only file from this repo you need to copy alongside the Streamlit app is -**`embeddings_v2.py`** (it's self-contained — no project-internal imports). Or -you can inline the few functions you actually use (see §6/§7 for the bare -minimum). - ---- - -## 6. Embedding a user query - -The model and dimension must match what's already in the DB -(`gemini-embedding-2-preview`, 768-d). **The contract is asymmetric and is -expressed in the prompt, not the config**: queries get a `task: search result -| query: …` prefix; documents go in as `title: … | text: …`. The -`EmbedContentConfig.task_type` field is *silently ignored* by -gemini-embedding-2 on the consumer API — don't set it. (Helper functions in -`embeddings_v2.py` apply the wrapping for you.) - -```python -import embeddings_v2 as e - -# Queries — generate_text_embedding auto-wraps with format_query(). -vec_text = e.generate_text_embedding("UAP over the Aegean") -vec_image = e.generate_image_embedding("./uploaded.jpg") # image-only: no text instruction -vec_both = e.generate_multimodal_embedding( - "./uploaded.jpg", - e.format_query("what is this"), # pre-wrap when there IS a text part -) -``` - -`embeddings_v2` also exports: - -- `format_document_text(title, body)` → `"title: {title} | text: {body}"` (use when storing). -- `format_query(query)` → `"task: search result | query: {query}"` (use when querying with a text part attached to media). - -Minimal inline version if you don't want to import `embeddings_v2`: - -```python -import os -from google import genai -from google.genai import types as gt - -client = genai.Client(api_key=os.environ["GEMINI_API_KEY"]) - -def embed_text(text: str) -> list[float]: - r = client.models.embed_content( - model="gemini-embedding-2-preview", - contents=f"task: search result | query: {text}", # wrap, not task_type= - config=gt.EmbedContentConfig(output_dimensionality=768), - ) - return list(r.embeddings[0].values) -``` - ---- - -## 7. Searching with pgvector - -`embeddings_v2.search_similar()` already does this and returns a list of -`SimilarityHit` dataclasses. If you want raw SQL: - -```sql -SELECT source_type, source_id, parent_id, start_seconds, end_seconds, - embedded_image_url, embedded_text, - 1 - (embedding <=> %s) AS similarity -FROM embeddings -WHERE user_id = %s::uuid - AND (%s::text IS NULL OR source_type = %s) - AND (embedding <=> %s) <= %s -- distance <= 1 - threshold -ORDER BY embedding <=> %s -LIMIT %s; -``` - -Params, in order: `query_vec, user_id, source_type_or_null, source_type_or_null, query_vec, (1 - threshold), query_vec, limit`. - -Don't forget `register_vector(conn)` from `pgvector.psycopg` after connecting — -without it psycopg can't bind `list[float]` to the `vector` type. - ---- - -## 8. Result interpretation (per source_type) - -### `video_chunk` -- `parent_id` → e.g. `dvids_1007706`. Strip the prefix to get the DVIDS asset id. -- `embedded_image_url` → the human DVIDS page, e.g. `https://www.dvidshub.net/video/1007706`. -- `start_seconds`, `end_seconds` → the chunk's offsets within the source video - (one video typically has multiple chunks; show the timestamp to the user). -- `embedded_text` → the caption that was attached at embed time: the - `Video Title` + `Description Blurb` from `uap-data_v2.csv`. -- DVIDS deep-link with timestamp: append `?t={int(start_seconds)}` to the page - URL (or use the local file with `st.video(local_path, start_time=int(start_seconds))`). - -### `audio_clip` -- Same `parent_id` shape but with audio DVIDS ids (1007870–1007879 range for - Release 2). -- `embedded_image_url` is set even though the asset is audio (it's the DVIDS - page URL — the column was reused as the canonical media URL for any kind). -- For long recordings (>80s — the model's audio input cap), the asset is - segmented into ≤75s pieces; one row per piece with its own start/end. - -### `pdf_page` -- `parent_id` is the doc slug (e.g. `dow-uap-d017`, `cia-uap-d001`, - `doe-uap-d001`, `doe-uap-d002`, `doe-uap-d003`). -- `source_id` is `{parent_id}:p{NNNN}` with the page number zero-padded to - 4 digits (e.g. `dow-uap-d017:p0017`). Parse with a tiny regex to surface - the page number in the UI. -- `embedded_image_url` is the whole-PDF URL on war.gov — there's no per-page - URL on the source site, so deep-linking to a specific page means opening - the PDF and scrolling. -- `embedded_text` is composed at embed time as: `{Agency} - {Title}` / - `Date: ... Location: ...` / `Page N of M.` / `Document context: {blurb}` / - `Page OCR: {ocr}`, capped at 8000 chars. The same string was paired with - the rendered page image in the multimodal embed call. -- `start_seconds` / `end_seconds` are NULL. -- A rendered page image lives locally at - `D:\divided\release_2\UAP_Release_2\pages\{slug}\page_NNNN.png` (150 dpi). - Display it directly with `st.image(local_path)`; link to - `embedded_image_url` to open the whole PDF on war.gov. - ---- - -## 9. Where the media files live - -The previous session saved every downloaded media file under the user's local -drive (set by them as the persistence target): - -``` -D:\divided\release_2\UAP_Release_2\ -├── videos\dvids_{id}.mp4 (49 files, normalized originals from DVIDS) -├── audio\dvids_{id}.{ext} (7 source MP4 wrappers + extracted .m4a tracks) -└── pages\{slug}\page_NNNN.png (PDF page renders at 150 dpi, e.g. - pages\dow-uap-d017\page_0017.png) -``` - -The page PNGs are generated by `ingest_pdf_pages.py` and are safe to delete and -re-generate from the source `release_2\{doc}\page_NNNN\page_NNNN.pdf` files. - -This matters for the Streamlit UI: - -- If the app runs on the same machine, you can pass the local path straight - into `st.video(path, start_time=...)` / `st.audio(...)` — that's the smoothest - playback experience and supports seeking. -- If the app runs elsewhere, link out to the DVIDS page (`embedded_image_url`). - Direct CloudFront URLs work for download but seeking via HTTP from the - browser is hit-or-miss. -- A third option: upload the local files to S3/R2/Vercel Blob and rewrite URLs. - Not done. - -If the file isn't found locally and the URL is the DVIDS page, **don't try to -embed the CloudFront MP4 directly in `st.video()`** — DVIDS' `/download/asset/` -endpoint is 403-gated, and the CloudFront URLs aren't stored in the DB. You'd -need to re-scrape the page (see the `scrape_media_url` helper in -`retry_release_2.py` if you want that pattern). - ---- - -## 10. Minimal working Streamlit app - -Drop this at `app.py` next to `embeddings_v2.py`, set the secrets, and run -`streamlit run app.py`. It covers text query, source-type filter, threshold -slider, and inline media playback with timestamp seeking. - -```python -import os -import re -from pathlib import Path - -import psycopg -import streamlit as st - -import embeddings_v2 as e - -USER_ID = "00000000-0000-0000-0000-000000000001" -MEDIA_ROOT = Path(r"D:\divided\release_2\UAP_Release_2") # change if elsewhere -SOURCE_TYPES = ("video_chunk", "audio_clip", "pdf_page") - -st.set_page_config(page_title="UAP Archive Semantic Search", layout="wide") - -# --- bootstrap --------------------------------------------------------------- -for k in ("DATABASE_URL", "GEMINI_API_KEY"): - if k in st.secrets: - os.environ.setdefault(k, st.secrets[k]) - if not os.environ.get(k): - st.error(f"Missing {k} — add it to .streamlit/secrets.toml") - st.stop() - -@st.cache_resource -def get_conn(): - return psycopg.connect(os.environ["DATABASE_URL"]) - -@st.cache_data(ttl=3600, show_spinner=False) -def embed_query_text(text: str) -> list[float]: - # generate_text_embedding auto-wraps with format_query() and drops task_type. - return e.generate_text_embedding(text) - -@st.cache_data(ttl=3600, show_spinner=False) -def embed_query_image(image_bytes: bytes, mime: str) -> list[float]: - import tempfile - suffix = "." + mime.split("/", 1)[1] - with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as f: - f.write(image_bytes) - path = f.name - try: - # image-only embed: same call for query and document side. - return e.generate_image_embedding(path) - finally: - os.unlink(path) - -def search(vec, *, source_type=None, release=None, limit=20, threshold=0.30): - # pgvector's psycopg adapter doesn't auto-cast list[float] to vector --- - # serialise to the textual '[a,b,c]' form and let Postgres cast. - vec_str = "[" + ",".join(f"{x:.6f}" for x in vec) + "]" - clauses = ["user_id = %s::uuid", "(embedding <=> %s::vector) <= %s"] - params = [USER_ID, vec_str, 1 - threshold] - if source_type: - clauses.append("source_type = %s") - params.append(source_type) - if release: - clauses.append("release = %s") - params.append(release) - sql = f""" - SELECT source_type, source_id, parent_id, start_seconds, end_seconds, - embedded_image_url, embedded_text, release, release_date, - 1 - (embedding <=> %s::vector) AS similarity - FROM embeddings - WHERE {' AND '.join(clauses)} - ORDER BY embedding <=> %s::vector - LIMIT %s - """ - ordered = [vec_str, *params, vec_str, limit] - with get_conn().cursor() as cur: - cur.execute(sql, ordered) - cols = [d.name for d in cur.description] - return [dict(zip(cols, r)) for r in cur.fetchall()] - -_PAGE_RE = re.compile(r"^(.+):p(\d+)$") - -def local_media_path(row: dict) -> Path | None: - st_type = row["source_type"] - if st_type == "video_chunk": - asset_id = row["parent_id"].removeprefix("dvids_") - p = MEDIA_ROOT / "videos" / f"dvids_{asset_id}.mp4" - return p if p.exists() else None - if st_type == "audio_clip": - asset_id = row["parent_id"].removeprefix("dvids_") - for ext in ("m4a", "mp3", "mp4", "wav", "aac", "ogg"): - p = MEDIA_ROOT / "audio" / f"dvids_{asset_id}.{ext}" - if p.exists(): - return p - return None - if st_type == "pdf_page": - m = _PAGE_RE.match(row["source_id"]) - if not m: - return None - slug, page_num = m.group(1), int(m.group(2)) - p = MEDIA_ROOT / "pages" / slug / f"page_{page_num:04d}.png" - return p if p.exists() else None - return None - -def page_number(row: dict) -> int | None: - if row["source_type"] != "pdf_page": - return None - m = _PAGE_RE.match(row["source_id"]) - return int(m.group(2)) if m else None - -# --- UI ---------------------------------------------------------------------- -st.title("UAP Archive — Semantic Search") -st.caption("Gemini 768-d embeddings, cosine similarity over Neon + pgvector.") - -with st.sidebar: - mode = st.radio("Query type", ["Text", "Image"], horizontal=True) - st_filter = st.selectbox("Source type", ["all", *SOURCE_TYPES]) - release_filter = st.selectbox("Release", ["all", "PURSUE_2"]) - threshold = st.slider("Min similarity", 0.0, 0.9, 0.30, 0.05) - limit = st.slider("Max results", 5, 50, 20) - -vec = None -if mode == "Text": - q = st.text_input("Search query", placeholder="e.g. spherical UAP over water") - if q: - with st.spinner("Embedding query…"): - vec = embed_query_text(q) -else: - up = st.file_uploader("Upload an image", type=["jpg", "jpeg", "png", "webp"]) - if up: - st.image(up, width=240) - with st.spinner("Embedding image…"): - vec = embed_query_image(up.getvalue(), up.type) - -if vec is None: - st.info("Enter a query or upload an image.") - st.stop() - -with st.spinner("Searching Neon…"): - rows = search( - vec, - source_type=None if st_filter == "all" else st_filter, - release=None if release_filter == "all" else release_filter, - limit=limit, - threshold=threshold, - ) - -if not rows: - st.warning("No matches above the similarity threshold. Try lowering it.") - st.stop() - -st.subheader(f"{len(rows)} result(s)") -for r in rows: - with st.container(border=True): - c1, c2 = st.columns([4, 1]) - with c1: - header = f"**[{r['parent_id']}]({r['embedded_image_url']})** · `{r['source_type']}` · sim **{r['similarity']:.3f}**" - page = page_number(r) - if page is not None: - header += f" · page {page}" - elif r["start_seconds"] is not None: - header += f" · {r['start_seconds']:.1f}s → {r['end_seconds']:.1f}s" - st.markdown(header) - if r["embedded_text"]: - st.write(r["embedded_text"][:600] + ("…" if len(r["embedded_text"]) > 600 else "")) - local = local_media_path(r) - if local and r["source_type"] == "video_chunk": - st.video(str(local), start_time=int(r["start_seconds"] or 0)) - elif local and r["source_type"] == "audio_clip": - st.audio(str(local), start_time=int(r["start_seconds"] or 0)) - elif local and r["source_type"] == "pdf_page": - st.image(str(local), use_container_width=True) - if r["embedded_image_url"]: - st.link_button("Open full PDF on war.gov", r["embedded_image_url"]) - elif r["embedded_image_url"]: - st.link_button("Open source", r["embedded_image_url"]) - with c2: - st.metric("similarity", f"{r['similarity']:.3f}") - st.caption(f"{r['release']} · {r['release_date']}") -``` - ---- - -## 11. Gotchas / things that will trip you up - -- **Pooled vs direct Neon endpoint.** The user's connection string in the - earlier session was the `-pooler` host. For a long-lived Streamlit process - that reuses one connection across many queries, psycopg3 will eventually - promote a statement to a *named* prepared statement (default - `prepare_threshold=5`), which PgBouncer in transaction-pooling mode cannot - hold across transactions. Use the **direct** endpoint (host without - `-pooler`) or set `prepare_threshold=None` on the connection. -- **Dimension must match.** The column is `VECTOR(768)`. Don't pass a 1536-dim - vector — it'll fail on the cast. If you ever switch to a different - `output_dimensionality`, you'll need to migrate the column. -- **Instruction-in-prompt, not `task_type=`.** gemini-embedding-2 silently - ignores `EmbedContentConfig.task_type` on the consumer API and instead - expects the task to be expressed *inside the content*. Wrap documents as - `title: {title} | text: {body}` (via `e.format_document_text(...)`) and - queries as `task: search result | query: {q}` (via `e.format_query(...)`, - applied automatically by `e.generate_text_embedding`). Skipping this - produces noticeably worse ranking — the previous version of this corpus - ranked NASA audio narratives above DOW UAP video clips on the query - "instantaneous acceleration" because the asymmetric format wasn't applied; - the re-embed with proper wrapping put `dvids_1007707` at ranks 1–4. - -- **Vertex-only config options.** Three `EmbedContentConfig` fields exist in - the SDK but are rejected by the consumer Gemini API - (`"