New commit for backend deployment: 2025-09-25_13-24-03

.gitignore

@@ -0,0 +1,96 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Virtual environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+venv/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Environment variables
+.env
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+# Data files
+*.csv
+*.xlsx
+*.pickle
+*.pkl
+
+# API keys and secrets
+secrets/
+*.key
+*.pem
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+logs/
+
+# Model files
+models/
+*.model
+*.h5
+*.pkl
+
+# Vector databases
+chroma_db/
+faiss_index/
+
+frontend/.next/
+frontend/node_modules/

Dockerfile

@@ -0,0 +1,16 @@
+# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.9
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,54 @@
+---
+title: Recipe Recommendation Chatbot API
+emoji: 🥘
+colorFrom: indigo
+colorTo: pink
+sdk: docker
+pinned: false
+license: mit
+---
+
+# Recipe Recommendation Chatbot
+
+A GenAI-powered chatbot that recommends recipes based on available ingredients using RAG (Retrieval Augmented Generation).
+
+## 🚀 Quick Start
+```bash
+# Clone repository
+git clone https://github.com/A3copilotprogram/PLG4-Recipe-Recommendation-Chatbot.git
+cd PLG4-Recipe-Recommendation-Chatbot
+
+# Install dependencies
+pip install -r requirements.txt
+
+# Run the chatbot
+python src/main.py
+```
+
+## 📁 Project Structure
+- `backend/` - FastAPI backend with RAG pipeline
+- `frontend/` - React frontend interface  
+- `data/` - Recipe datasets and embeddings
+- `docs/` - Project documentation
+- `notebooks/` - Jupyter notebooks for exploration
+- `tests/` - Unit and integration tests
+
+## 📚 Documentation
+
+### Quick Start Guides
+- **[Backend Setup](./backend/README.md)** - FastAPI server setup and configuration
+- **[Frontend Setup](./frontend/README.md)** - React app development
+
+### Troubleshooting
+- **[Embedding Issues](./backend/docs/embedding-troubleshooting.md)** - Fix common dimension mismatch errors
+- **[Documentation Index](./backend/docs/README.md)** - Complete documentation overview
+
+### Architecture
+- **[System Architecture](./docs/architecture.md)** - High-level system design
+- **[API Documentation](./backend/docs/api-documentation.md)** - Detailed API reference
+
+## 🤝 Contributing
+See [CONTRIBUTING.md](docs/CONTRIBUTING.md) for ways of working and contribution guidelines.
+
+## 👥 Team
+GenAI PLG 4 - Andela Community Program

app.py

@@ -0,0 +1 @@
+from backend.app import app

backend/.env.example

@@ -0,0 +1,105 @@
+# ===========================================
+# Recipe Recommendation Bot - Environment Configuration
+# ===========================================
+
+# Server Configuration
+PORT=8080
+HOST=0.0.0.0
+ENVIRONMENT=development
+DEBUG=true
+LANGCHAIN_DEBUG=true
+
+# CORS Configuration
+CORS_ORIGINS=["http://localhost:3000","http://localhost:5173","http://localhost:8000"]
+CORS_ALLOW_CREDENTIALS=true
+CORS_ALLOW_METHODS=["GET","POST","PUT","DELETE","OPTIONS"]
+CORS_ALLOW_HEADERS=["*"]
+
+# ===========================================
+# LLM & Embedding Provider Configuration
+# ===========================================
+# Supported providers: openai, google, huggingface, ollama
+# This provider will be used for both LLM and embeddings
+
+LLM_PROVIDER=google
+EMBEDDING_PROVIDER=google
+
+# OpenAI Configuration
+# Use only if LLM_PROVIDER or EMBEDDING_PROVIDER is set to 'openai'
+OPENAI_API_KEY=YOUR_OPENAI_API_KEY_HERE
+OPENAI_MODEL=gpt-5-nano
+OPENAI_TEMPERATURE=0.7
+OPENAI_MAX_TOKENS=1000
+
+# Google AI Configuration (Gemini)
+# Use only if LLM_PROVIDER or EMBEDDING_PROVIDER is set to 'google'
+GOOGLE_API_KEY=YOUR_GOOGLE_API_KEY_HERE
+GOOGLE_MODEL=gemini-2.0-flash
+GOOGLE_TEMPERATURE=0.7
+GOOGLE_MAX_TOKENS=1000
+
+# Hugging Face Configuration
+# Use only if LLM_PROVIDER or EMBEDDING_PROVIDER is set to 'huggingface'
+HUGGINGFACE_API_TOKEN=YOUR_HUGGINGFACE_API_TOKEN_HERE
+HUGGINGFACE_MODEL=deepseek-ai/DeepSeek-V3.1
+HUGGINGFACE_API_URL=https://api-inference.huggingface.co/models/
+HUGGINGFACE_USE_API=true
+HUGGINGFACE_USE_GPU=false
+
+# Ollama Configuration (local inference)
+# Use only if LLM_PROVIDER or EMBEDDING_PROVIDER is set to 'ollama'
+OLLAMA_BASE_URL=http://localhost:11434
+OLLAMA_MODEL=llama3.1:8b
+OLLAMA_TEMPERATURE=0.7
+
+
+# ===========================================
+# Vector Store Configuration
+# ===========================================
+# Supported stores: chromadb, mongodb
+VECTOR_STORE_PROVIDER=mongodb
+
+# ChromaDB Configuration
+DB_PATH=./data/chromadb
+DB_COLLECTION_NAME=recipes
+DB_PERSIST_DIRECTORY=./data/chromadb_persist
+# Set to true to delete and recreate DB on startup (useful for adding new recipes)
+DB_REFRESH_ON_START=false
+
+# MongoDB Atlas Configuration (for vector search)
+# Provide your connection string and collection settings when using MongoDB
+MONGODB_URI=mongodb+srv://<username>:<password>@<cluster>.mongodb.net/?retryWrites=true&w=majority&appName=<AppName>
+MONGODB_DATABASE=food_recommendation
+MONGODB_COLLECTION=AI_DB
+MONGODB_INDEX_NAME=foodInstructionIndex
+MONGODB_VECTOR_FIELD=ingredients_emb
+MONGODB_TEXT_FIELD=title
+MONGODB_SIMILARITY_METRIC=dotProduct
+MONGODB_NUM_CANDIDATES=100
+
+
+# ===========================================
+# Model Configuration
+# ===========================================
+# The LLM_PROVIDER setting above controls both LLM and embedding models
+
+# OpenAI Models
+OPENAI_EMBEDDING_MODEL=text-embedding-3-small
+
+# Google Models
+GOOGLE_EMBEDDING_MODEL=models/embedding-001
+
+# HuggingFace Models
+HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+
+# Ollama Models
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text:v1.5
+
+
+# ===========================================
+# Logging Configuration
+# ===========================================
+LOG_LEVEL=INFO
+LOG_FORMAT=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+LOG_FILE=./logs/app.log
+

backend/.gitignore

@@ -0,0 +1,141 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Application specific
+vector_store/
+logs/
+*.log
+.DS_Store
+node_modules/
+
+# Data folder - ignore everything except sample recipe
+data/*
+!data/sample_recipes.json
+!data/recipes

backend/Dockerfile

@@ -0,0 +1,16 @@
+# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.9
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

backend/README.md

@@ -0,0 +1,462 @@
+# Recipe Recommendation Chatbot - Backend API
+
+Backend for AI-powered recipe recommendation system built with FastAPI, featuring RAG (Retrieval-Augmented Generation) capabilities, conversational memory, and multi-provider LLM support.
+
+## 🚀 Quick Start
+
+### Prerequisites
+- Python 3.9+
+- pip or poetry
+- API keys for your chosen LLM provider (OpenAI, Google, or HuggingFace)
+
+### Installation
+
+1. **Clone and navigate to backend**
+   ```bash
+   git clone <repository-url>
+   cd PLG4-Recipe-Recommendation-Chatbot/backend
+   ```
+
+2. **Install dependencies**
+   ```bash
+   pip install -r requirements.txt
+   ```
+   > 💡 **Note**: Some packages are commented out by default to keep the installation lightweight:
+   > - **HuggingFace dependencies** (`transformers`, `accelerate`, `sentence-transformers`) - Uncomment if using HuggingFace models
+   > - **sentence-transformers** (~800MB) - Uncomment for HuggingFace embeddings
+
+3. **Configure environment**
+   ```bash
+   cp .env.example .env
+   # Edit .env with your API keys and configuration
+   ```
+
+4. **Run the server**
+   ```bash
+   # Development mode with auto-reload
+   uvicorn app:app --reload --host 127.0.0.1 --port 8080
+   
+   # Or production mode
+   uvicorn app:app --host 127.0.0.1 --port 8080
+   ```
+
+5. **Test the API**
+   ```bash
+   curl http://localhost:8080/health
+   ```
+
+6. **HuggingFace Spaces deployment**
+   ```
+   sh deploy-to-hf.sh <remote>
+   ``` 
+   where <remote> points to the HuggingFace Spaces repository
+
+## 📁 Project Structure
+
+```
+backend/
+├── app.py                 # FastAPI application entry point
+├── requirements.txt       # Python dependencies
+├── .env.example          # Environment configuration template
+├── .gitignore            # Git ignore rules
+│
+├── config/               # Configuration modules
+│   ├── __init__.py
+│   ├── settings.py       # Application settings
+│   ├── database.py       # Database configuration
+│   └── logging_config.py # Logging setup
+│
+├── services/             # Core business logic
+│   ├── __init__.py
+│   ├── llm_service.py    # LLM and RAG pipeline
+│   └── vector_store.py   # Vector database management
+│
+├── data/                 # Data storage
+│   ├── recipes/          # Recipe JSON files
+│   │   └── recipe.json   # Sample recipe data
+│   └── chromadb_persist/ # ChromaDB persistence
+│
+├── logs/                 # Application logs
+│   └── recipe_bot.log    # Main log file
+│
+├── docs/                 # Documentation
+│   ├── model-selection-guide.md      # 🎯 Complete model selection & comparison guide
+│   ├── model-quick-reference.md      # ⚡ Quick model switching commands  
+│   ├── chromadb_refresh.md           # ChromaDB refresh guide
+│   ├── opensource-llm-configuration.md  # Open source LLM setup guide
+│   ├── logging_guide.md              # Logging documentation
+│   ├── optimal_recipes_structure.md  # Recipe data structure guide
+│   ├── sanitization_guide.md         # Input sanitization guide
+│   └── unified-provider-configuration.md  # Unified provider approach guide
+│
+└── utils/                # Utility functions
+    └── __init__.py
+```
+
+## ⚙️ Configuration
+
+### Environment Variables
+
+Copy `.env.example` to `.env` and configure the following:
+
+> 🎯 **Unified Provider Approach**: The `LLM_PROVIDER` setting controls both LLM and embedding models, preventing configuration mismatches. See [`docs/unified-provider-configuration.md`](docs/unified-provider-configuration.md) for details.
+
+#### **Server Configuration**
+```bash
+PORT=8000                 # Server port
+HOST=0.0.0.0             # Server host
+ENVIRONMENT=development   # Environment mode
+DEBUG=true               # Debug mode
+```
+
+#### **Provider Configuration**
+Choose one provider for both LLM and embeddings (unified approach):
+
+> 🎯 **NEW: Complete Model Selection Guide**: For detailed comparisons of all models (OpenAI, Google, Anthropic, Ollama, HuggingFace) including latest 2025 models, performance metrics, costs, and scenario-based recommendations, see [`docs/model-selection-guide.md`](docs/model-selection-guide.md)
+
+> ⚡ **Quick Reference**: For one-command model switching, see [`docs/model-quick-reference.md`](docs/model-quick-reference.md)
+
+**OpenAI (Best Value & Latest Models)**
+```bash
+LLM_PROVIDER=openai
+OPENAI_API_KEY=your_openai_api_key_here
+OPENAI_MODEL=gpt-5-nano             # 🎯 BEST VALUE: $1/month for 30K queries - Modern GPT-5 at nano price
+# Alternatives:
+# - gpt-4o-mini                     # Proven choice: $4/month for 30K queries
+# - gpt-5                           # Premium: $20/month unlimited (Plus plan)
+OPENAI_EMBEDDING_MODEL=text-embedding-3-small # Used automatically
+```
+
+**Google Gemini (Best Free Tier)**
+```bash
+LLM_PROVIDER=google
+GOOGLE_API_KEY=your_google_api_key_here
+GOOGLE_MODEL=gemini-2.5-flash       # 🎯 RECOMMENDED: Excellent free tier, then $2/month
+# Alternatives:
+# - gemini-2.0-flash-lite           # Ultra budget: $0.90/month for 30K queries
+# - gemini-2.5-pro                  # Premium: $25/month for 30K queries
+GOOGLE_EMBEDDING_MODEL=models/embedding-001 # Used automatically
+```
+
+**Anthropic Claude (Best Quality-to-Cost)**
+```bash
+LLM_PROVIDER=anthropic
+ANTHROPIC_API_KEY=your_anthropic_api_key_here
+ANTHROPIC_MODEL=claude-3-5-haiku-20241022  # 🎯 BUDGET WINNER: $4/month for 30K queries
+# Alternatives:
+# - claude-3-5-sonnet-20241022      # Production standard: $45/month for 30K queries
+# - claude-3-opus-20240229          # Premium quality: $225/month for 30K queries
+ANTHROPIC_EMBEDDING_MODEL=voyage-large-2 # Used automatically
+```
+
+**Ollama (Best for Privacy/Self-Hosting)**
+```bash
+LLM_PROVIDER=ollama
+OLLAMA_BASE_URL=http://localhost:11434
+OLLAMA_MODEL=llama3.1:8b            # 🎯 YOUR CURRENT: 4.7GB download, 8GB RAM, excellent balance
+# New alternatives: 
+# - deepseek-r1:7b                  # Breakthrough reasoning: 4.7GB download, O1-level performance
+# - codeqwen:7b                     # Structured data expert: 4.2GB download, excellent for recipes
+# - gemma3:4b                       # Resource-efficient: 3.3GB download, 6GB RAM
+# - mistral-nemo:12b                # Balanced performance: 7GB download, 12GB RAM
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text # Used automatically
+```
+
+**HuggingFace (Downloadable Models Only - APIs Unreliable)**
+```bash
+LLM_PROVIDER=ollama  # Use Ollama to run HuggingFace models locally
+OLLAMA_MODEL=codeqwen:7b             # 🎯 RECOMMENDED: Download HF models via Ollama for reliability
+# Other downloadable options:
+# - mistral-nemo:12b                # Mistral's balanced model
+# - nous-hermes2:10.7b              # Fine-tuned for instruction following
+# - openhermes2.5-mistral:7b        # Community favorite
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text # Used automatically
+```
+> ⚠️ **Important Change**: HuggingFace APIs have proven unreliable for production. We now recommend downloading HuggingFace models locally via Ollama for consistent performance.
+> ⚠️ **HuggingFace Update**: HuggingFace dependencies are no longer required as we recommend using downloadable models via Ollama instead of unreliable APIs. For local HuggingFace models, use Ollama which provides better reliability and performance.
+
+> 📖 **Local Model Setup**: See [`docs/opensource-llm-configuration.md`](docs/opensource-llm-configuration.md) for GPU setup, model selection, and performance optimization with Ollama.
+
+> 💡 **Unified Provider**: The `LLM_PROVIDER` setting automatically configures both the LLM and embedding models, ensuring consistency and preventing mismatched configurations.
+
+#### **Vector Store Configuration**
+Choose between ChromaDB (local) or MongoDB Atlas:
+
+**ChromaDB (Default)**
+```bash
+VECTOR_STORE_PROVIDER=chromadb
+DB_COLLECTION_NAME=recipes
+DB_PERSIST_DIRECTORY=./data/chromadb_persist
+# Set to true to delete and recreate DB on startup (useful for adding new recipes)
+DB_REFRESH_ON_START=false
+```
+
+**MongoDB Atlas**
+```bash
+VECTOR_STORE_PROVIDER=mongodb
+MONGODB_URI=mongodb+srv://username:password@cluster.mongodb.net/
+MONGODB_DATABASE=recipe_bot
+MONGODB_COLLECTION=recipes
+```
+
+#### **Embedding Configuration**
+```bash
+# Embedding provider automatically matches LLM_PROVIDER (unified approach)
+# No separate configuration needed - handled automatically based on LLM_PROVIDER setting
+```
+
+> 💡 **Unified Provider**: The `LLM_PROVIDER` setting automatically configures both the LLM and embedding models, ensuring consistency and preventing mismatched configurations. See [`docs/model-selection-guide.md`](docs/model-selection-guide.md) for all available options.
+
+## 🛠️ API Endpoints
+
+### Core Endpoints
+
+#### **Health Check**
+```bash
+GET /health
+```
+Returns service health and configuration status.
+
+#### **Chat with RAG**
+```bash
+POST /chat
+Content-Type: application/json
+
+{
+  "message": "What chicken recipes do you have?"
+}
+```
+Full conversational RAG pipeline with memory and vector retrieval.
+
+#### **Simple Demo**
+```bash
+GET /demo?prompt=Tell me about Italian cuisine
+```
+Simple LLM completion without RAG for testing.
+
+#### **Clear Memory**
+```bash
+POST /clear-memory
+```
+Clears conversation memory for fresh start.
+
+### Example Requests
+
+**Chat Request:**
+```bash
+curl -X POST "http://localhost:8080/chat" 
+  -H "Content-Type: application/json" 
+  -d '{"message": "What are some quick breakfast recipes?"}'
+```
+
+**Demo Request:**
+```bash
+curl "http://localhost:8080/demo?prompt=What%20is%20your%20favorite%20pasta%20dish?"
+```
+
+## 🏗️ Architecture
+
+### Core Components
+
+#### **LLM Service** (`services/llm_service.py`)
+- **ConversationalRetrievalChain**: Main RAG pipeline with memory
+- **Simple Chat Completion**: Direct LLM responses without RAG
+- **Multi-provider Support**: OpenAI, Google, HuggingFace
+- **Conversation Memory**: Persistent chat history
+
+#### **Vector Store Service** (`services/vector_store.py`)
+- **ChromaDB Integration**: Local vector database
+- **MongoDB Atlas Support**: Cloud vector search
+- **Document Loading**: Automatic recipe data ingestion
+- **Embedding Management**: Multi-provider embedding support
+
+#### **Configuration System** (`config/`)
+- **Settings Management**: Environment-based configuration
+- **Database Configuration**: Vector store setup
+- **Logging Configuration**: Structured logging with rotation
+
+### Data Flow
+
+1. **User Query** → FastAPI endpoint
+2. **RAG Pipeline** → Vector similarity search
+3. **Context Retrieval** → Top-k relevant recipes
+4. **LLM Generation** → Context-aware response
+5. **Memory Storage** → Conversation persistence
+6. **Response** → JSON formatted reply
+
+## 📊 Logging
+
+Comprehensive logging system with:
+
+- **File Rotation**: 10MB max size, 5 backups
+- **Structured Format**: Timestamps, levels, source location
+- **Emoji Indicators**: Visual status indicators
+- **Error Tracking**: Full stack traces for debugging
+
+**Log Levels:**
+- 🚀 **INFO**: Normal operations
+- ⚠️ **WARNING**: Non-critical issues
+- ❌ **ERROR**: Failures with stack traces
+- 🔧 **DEBUG**: Detailed operation steps
+
+**Log Location:** `./logs/recipe_bot.log`
+
+## 📁 Data Management
+
+### Recipe Data
+- **Location**: `./data/recipes/`
+- **Format**: JSON files with structured recipe data
+- **Schema**: title, ingredients, directions, tags
+- **Auto-loading**: Automatic chunking and vectorization
+
+### Vector Storage
+- **ChromaDB**: Local persistence in `./data/chromadb_persist/`
+- **MongoDB**: Cloud-based vector search
+- **Embeddings**: Configurable embedding models
+- **Retrieval**: Top-k similarity search (k=25)
+
+## 🔧 Development
+
+### Running in Development
+```bash
+# Install dependencies
+pip install -r requirements.txt
+
+# Set up environment
+cp .env.example .env
+# Configure your API keys
+
+# Run with auto-reload
+uvicorn app:app --reload --host 127.0.0.1 --port 8080
+```
+
+### Testing Individual Components
+```bash
+# Test vector store
+python -c "from services.vector_store import vector_store_service; print('Vector store initialized')"
+
+# Test LLM service
+python -c "from services.llm_service import llm_service; print('LLM service initialized')"
+```
+
+### Adding New Recipes
+1. Add JSON files to `./data/recipes/`
+2. Set `DB_REFRESH_ON_START=true` in `.env` file
+3. Restart the application (ChromaDB will be recreated)
+4. Set `DB_REFRESH_ON_START=false` to prevent repeated deletion
+5. New recipes are now available for search
+
+**Quick refresh:**
+```bash
+# Enable refresh, restart, then disable
+echo "DB_REFRESH_ON_START=true" >> .env
+uvicorn app:app --reload --host 127.0.0.1 --port 8080
+# After startup completes:
+sed -i 's/DB_REFRESH_ON_START=true/DB_REFRESH_ON_START=false/' .env
+```
+
+## 🚀 Production Deployment
+
+### Environment Setup
+```bash
+ENVIRONMENT=production
+DEBUG=false
+LOG_LEVEL=INFO
+```
+
+### Docker Deployment
+The backend is containerized and ready for deployment on platforms like Hugging Face Spaces.
+
+### Security Features
+- **Environment Variables**: Secure API key management
+- **CORS Configuration**: Frontend integration protection  
+- **Input Sanitization**: Context-appropriate validation for recipe queries
+  - XSS protection through HTML encoding
+  - Length validation (1-1000 characters)
+  - Basic harmful pattern removal
+  - Whitespace normalization
+- **Pydantic Validation**: Type safety and automatic sanitization
+- **Structured Error Handling**: Safe error responses without data leaks
+
+## 🛠️ Troubleshooting
+
+### Common Issues
+
+**Vector store initialization fails**
+- Check API keys for embedding provider
+- Verify data folder contains recipe files
+- Check ChromaDB permissions
+
+**LLM service fails**
+- Verify API key configuration
+- Check provider-specific requirements
+- Review logs for detailed error messages
+
+**HuggingFace model import errors**
+- HuggingFace APIs have proven unreliable for production use
+- **Recommended**: Use Ollama to run HuggingFace models locally instead:
+  ```bash
+  # Install and run HuggingFace models via Ollama
+  ollama pull codeqwen:7b
+  ollama pull mistral-nemo:12b
+  # Set LLM_PROVIDER=ollama in .env
+  ```
+- For legacy HuggingFace API setup, uncomment dependencies in `requirements.txt` (not recommended)
+- For detailed model comparisons, see [`docs/model-selection-guide.md`](docs/model-selection-guide.md)
+
+**Memory issues**
+```bash
+# Clear conversation memory
+curl -X POST http://localhost:8080/clear-memory
+```
+
+### Debug Mode
+Set `DEBUG=true` in `.env` for detailed logging and error traces.
+
+### Log Analysis
+Check `./logs/recipe_bot.log` for detailed operation logs with emoji indicators for quick status identification.
+
+## 📚 Documentation
+
+### Troubleshooting Guides
+- **[Embedding Troubleshooting](./docs/embedding-troubleshooting.md)** - Quick fixes for common embedding dimension errors
+- **[Embedding Compatibility Guide](./docs/embedding-compatibility-guide.md)** - Comprehensive guide to embedding models and dimensions
+- **[Logging Guide](./docs/logging_guide.md)** - Understanding the logging system
+
+### Technical Guides
+- **[Architecture Documentation](./docs/architecture.md)** - System architecture overview
+- **[API Documentation](./docs/api-documentation.md)** - Detailed API reference
+- **[Deployment Guide](./docs/deployment.md)** - Production deployment instructions
+
+### Common Issues
+- **Dimension mismatch errors**: See [Embedding Troubleshooting](./docs/embedding-troubleshooting.md)
+- **Model loading issues**: Check provider configuration in `.env`
+- **Database connection problems**: Verify MongoDB/ChromaDB settings
+
+## 📚 Dependencies
+
+### Core Dependencies
+- **FastAPI**: Modern web framework
+- **uvicorn**: ASGI server
+- **pydantic**: Data validation
+- **python-dotenv**: Environment management
+
+### AI/ML Dependencies
+- **langchain**: LLM framework and chains
+- **langchain-openai**: OpenAI integration
+- **langchain-google-genai**: Google AI integration
+- **sentence-transformers**: Embedding models
+- **chromadb**: Vector database
+- **pymongo**: MongoDB integration
+
+### Optional Dependencies
+- **langchain-huggingface**: HuggingFace integration
+- **torch**: PyTorch for local models
+
+## 📄 License
+
+This project is part of the PLG4 Recipe Recommendation Chatbot system.
+
+---
+
+For more detailed documentation, check the `docs/` folder or visit the API documentation at `http://localhost:8080/docs` when running the server.

backend/app.py

@@ -0,0 +1,193 @@
+from backend.utils.request_dto.chat_response import ChatResponse
+from backend.utils.request_dto.scrape_request import ScrapeRequest
+from backend.utils.types import ChatMessage
+from fastapi import FastAPI, HTTPException, BackgroundTasks, Header
+from fastapi.middleware.cors import CORSMiddleware
+import os
+from typing import Type
+from fastapi.middleware.cors import CORSMiddleware
+from data_minning.dto.stream_opts import StreamOptions
+from data_minning.base_scrapper import BaseRecipeScraper, JsonArraySink, MongoSink
+from data_minning.all_nigerian_recipe_scraper import AllNigerianRecipesScraper
+from data_minning.yummy_medley_scraper import YummyMedleyScraper
+from backend.config.settings import settings
+from backend.config.logging_config import setup_default_logging, get_logger
+from backend.utils.sanitization import sanitize_user_input
+from backend.services.vector_store import vector_store_service
+# Setup logging first, before importing services
+setup_default_logging()
+logger = get_logger("app")
+
+# Import services after logging is configured
+from backend.services.llm_service import llm_service
+
+SCRAPERS: dict[str, Type[BaseRecipeScraper]] = {
+    "yummy": YummyMedleyScraper,
+    "anr": AllNigerianRecipesScraper,
+}
+
+app = FastAPI(
+    title="Recipe Recommendation Bot API",
+    description="AI-powered recipe recommendation system with RAG capabilities",
+    version="1.0.0"
+)
+
+logger.info("🚀 Starting Recipe Recommendation Bot API")
+logger.info(f"Environment: {settings.ENVIRONMENT}")
+logger.info(f"Provider: {settings.get_llm_config()['provider']} (LLM + Embeddings)")
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.CORS_ORIGINS or ["*"],
+    allow_credentials=settings.CORS_ALLOW_CREDENTIALS or True,
+    allow_methods=settings.CORS_ALLOW_METHODS or ["*"],
+    allow_headers=settings.CORS_ALLOW_HEADERS or ["*"],
+)
+
+# Remove OpenAI direct setup - now handled by LLM service
+# if settings.OPENAI_API_KEY:
+#     openai.api_key = settings.OPENAI_API_KEY
+
+@app.get("/")
+def index():
+    logger.info("📡 Root endpoint accessed")
+    return {
+        "message": "Recipe Recommendation Bot API",
+        "version": "1.0.0",
+        "status": "running"
+    }
+
+@app.get("/health")
+def health_check():
+    logger.info("🏥 Health check endpoint accessed")
+    return {
+        "status": "healthy",
+        "environment": settings.ENVIRONMENT,
+        "llm_service_initialized": llm_service is not None
+    }
+
+@app.post("/chat", response_model=ChatResponse)
+async def chat(chat_message: ChatMessage):
+    """Main chatbot endpoint - Recipe recommendation with ConversationalRetrievalChain"""
+    try:
+        # Message is already sanitized by the Pydantic validator
+        # Find the last user message in the messages list
+        last_user_message = chat_message.get_latest_message()
+        if not last_user_message:
+            raise ValueError("No valid user message found")
+        user_text = last_user_message.parts[0].text
+
+        response_text = llm_service.ask_question(user_text)
+        return ChatResponse(response=response_text)
+        
+    except ValueError as e:
+        # Handle validation/sanitization errors
+        logger.warning(f"⚠️ Invalid input received: {str(e)}")
+        raise HTTPException(status_code=400, detail=f"Invalid input: {str(e)}")
+        
+    except Exception as e:
+        logger.error(f"❌ Chat service error: {str(e)}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Chat service error: {str(e)}")
+
+@app.get("/demo")
+def demo(prompt: str = "What recipes do you have?"):
+    """Demo endpoint - uses simple chat completion without RAG"""
+    logger.info(f"🎯 Demo request: '{prompt[:50]}...'")
+    
+    try:
+        # Sanitize the demo prompt using the same sanitization method
+        sanitized_prompt = sanitize_user_input(prompt)        
+        response_text = llm_service.simple_chat_completion(sanitized_prompt)
+        return {"prompt": sanitized_prompt, "reply": response_text}
+        
+    except ValueError as e:
+        # Handle validation/sanitization errors
+        logger.warning(f"⚠️ Invalid demo prompt: {str(e)}")
+        return {"error": f"Invalid prompt: {str(e)}", "prompt": prompt}
+        
+    except Exception as e:
+        logger.error(f"❌ Demo endpoint error: {str(e)}", exc_info=True)
+        return {"error": f"Failed to get response: {str(e)}"}
+
+@app.post("/clear-memory")
+def clear_conversation_memory():
+    """Clear conversation memory"""
+    logger.info("🧹 Memory clear request received")
+    
+    try:
+        success = llm_service.clear_memory()
+        
+        if success:
+            logger.info("✅ Conversation memory cleared successfully")
+            return {"status": "success", "message": "Conversation memory cleared"}
+        else:
+            logger.warning("⚠️ Memory clear operation failed")
+            return {"status": "failed", "message": "Failed to clear conversation memory"}
+            
+    except Exception as e:
+        logger.error(f"❌ Memory clear error: {str(e)}", exc_info=True)
+        return {"status": "error", "message": str(e)}
+
+
+
+def run_job(job_id: str, site: str, limit: int, output_type: str):
+    '''
+    Background job to run the scraper
+    Uses global JOBS dict to track status
+    Outputs to JSON file or MongoDB based on output_type
+    '''
+    s = SCRAPERS[site]()
+    s.embedder = vector_store_service._create_sentence_transformer_wrapper("sentence-transformers/all-MiniLM-L6-v2")
+    s.embedding_fields = [(("title", "ingredients", "instructions"), "recipe_emb")]
+    sink = None
+    if output_type == "json":
+        sink = JsonArraySink("./data/recipes_unified.json")
+    elif output_type == "mongo":
+        sink = MongoSink() if os.getenv("MONGODB_URI") else None
+
+    stream_opts = StreamOptions(
+            delay=0.3,
+            limit=500,
+            batch_size=limit,
+            resume_file="recipes.resume",
+            progress_callback=make_progress_cb(job_id),
+        )
+    try:
+        JOBS[job_id] = {"status": "running", "count": 0}
+        s.stream( sink=sink, options=stream_opts)
+        JOBS[job_id]["status"] = "done"
+    except Exception as e:
+        JOBS[job_id] = {"status": "error", "error": str(e)}
+
+def make_progress_cb(job_id: str):
+    ''' Create a progress callback to update JOBS dict 
+    '''
+    def _cb(n: int):
+        JOBS[job_id]["count"] = n
+    return _cb
+
+
+
+
+
+# super-lightweight in-memory job store (reset on restart)
+JOBS: dict[str, any] = {}
+
+@app.post("/scrape")
+def scrape(body: ScrapeRequest, background: BackgroundTasks, x_api_key: str = Header(None)):
+    if body.site not in SCRAPERS:
+        raise HTTPException(status_code=400, detail="Unknown site")
+
+    job_id = f"{body.site}-{os.urandom(4).hex()}"
+    # use thread via BackgroundTasks to avoid blocking the request
+    background.add_task(run_job, job_id, body.site, body.limit, body.output_type)
+    return {"job_id": job_id, "status": "queued"}
+
+@app.get("/jobs/{job_id}")
+def job_status(job_id: str):
+    return JOBS.get(job_id, {"status": "unknown"})
+
+@app.get("/jobs")
+def list_jobs():
+    return JOBS

backend/config/database.py

@@ -0,0 +1,61 @@
+# Database and vector store configuration
+import os
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+class DatabaseSettings:
+    """Simple database settings class that reads environment variables directly"""
+    
+    def __init__(self):
+        # ===========================================
+        # Vector Store Configuration
+        # ===========================================
+        self.VECTOR_STORE_PROVIDER = os.getenv("VECTOR_STORE_PROVIDER", "chromadb")
+        
+        # ChromaDB Configuration
+        self.DB_PATH = os.getenv("DB_PATH", "./data/chromadb")
+        self.DB_COLLECTION_NAME = os.getenv("DB_COLLECTION_NAME", "recipes")
+        self.DB_PERSIST_DIRECTORY = os.getenv("DB_PERSIST_DIRECTORY", "./data/chromadb_persist")
+        self.DB_REFRESH_ON_START = os.getenv("DB_REFRESH_ON_START", "false").lower() == "true"
+        
+        # MongoDB Atlas Configuration
+        self.MONGODB_URI = os.getenv("MONGODB_URI")
+        self.MONGODB_DATABASE = os.getenv("MONGODB_DATABASE", "recipe_bot")
+        self.MONGODB_COLLECTION = os.getenv("MONGODB_COLLECTION", "recipes")
+        self.MONGODB_INDEX_NAME = os.getenv("MONGODB_INDEX_NAME", "vector_index")
+        self.MONGODB_VECTOR_FIELD = os.getenv("MONGODB_VECTOR_FIELD", "embedding")
+        self.MONGODB_TEXT_FIELD = os.getenv("MONGODB_TEXT_FIELD", "text")
+        self.MONGODB_SIMILARITY_METRIC = os.getenv("MONGODB_SIMILARITY_METRIC", "cosine")
+        self.MONGODB_NUM_CANDIDATES = int(os.getenv("MONGODB_NUM_CANDIDATES", "50"))
+
+    def get_vector_store_config(self):
+        """Get vector store configuration based on selected provider"""
+        if self.VECTOR_STORE_PROVIDER == "chromadb":
+            return {
+                "provider": "chromadb",
+                "path": self.DB_PATH,
+                "collection_name": self.DB_COLLECTION_NAME,
+                "persist_directory": self.DB_PERSIST_DIRECTORY,
+                "refresh_on_start": self.DB_REFRESH_ON_START
+            }
+        elif self.VECTOR_STORE_PROVIDER == "mongodb":
+            if not self.MONGODB_URI:
+                raise ValueError("MongoDB URI is required when using MongoDB Atlas as vector store")
+            return {
+                "provider": "mongodb",
+                "uri": self.MONGODB_URI,
+                "database": self.MONGODB_DATABASE,
+                "collection_name": self.MONGODB_COLLECTION,
+                "index_name": self.MONGODB_INDEX_NAME,
+                "vector_field": self.MONGODB_VECTOR_FIELD,
+                "text_field": self.MONGODB_TEXT_FIELD,
+                "similarity_metric": self.MONGODB_SIMILARITY_METRIC,
+                "num_candidates": self.MONGODB_NUM_CANDIDATES
+            }
+        else:
+            raise ValueError(f"Unsupported vector store provider: {self.VECTOR_STORE_PROVIDER}")
+
+# Create global database settings instance
+db_settings = DatabaseSettings()

backend/config/logging_config.py

@@ -0,0 +1,82 @@
+# Logging Configuration
+import logging
+import logging.handlers
+import sys
+from pathlib import Path
+from typing import Optional
+
+def setup_logging(
+    log_level: str = "INFO",
+    log_file: Optional[str] = None,
+    enable_console: bool = True,
+    max_file_size: int = 10 * 1024 * 1024,  # 10MB
+    backup_count: int = 5
+) -> logging.Logger:
+    """
+    Setup centralized logging configuration
+    
+    Args:
+        log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+        log_file: Path to log file (optional)
+        enable_console: Whether to enable console logging
+        max_file_size: Maximum log file size in bytes
+        backup_count: Number of backup files to keep
+    
+    Returns:
+        Configured logger instance
+    """
+    
+    # Create logs directory if it doesn't exist
+    if log_file:
+        log_path = Path(log_file)
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+    
+    # Configure root logger
+    logger = logging.getLogger("recipe_bot")
+    logger.setLevel(getattr(logging, log_level.upper()))
+    
+    # Clear existing handlers
+    logger.handlers.clear()
+    
+    # Create formatter
+    formatter = logging.Formatter(
+        fmt="%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S"
+    )
+    
+    # Console handler
+    if enable_console:
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setLevel(getattr(logging, log_level.upper()))
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+    
+    # File handler with rotation
+    if log_file:
+        file_handler = logging.handlers.RotatingFileHandler(
+            filename=log_file,
+            maxBytes=max_file_size,
+            backupCount=backup_count,
+            encoding='utf-8'
+        )
+        file_handler.setLevel(getattr(logging, log_level.upper()))
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+    
+    # Prevent duplicate logs
+    logger.propagate = False
+    
+    return logger
+
+def get_logger(name: str) -> logging.Logger:
+    """Get a child logger with the specified name"""
+    return logging.getLogger(f"recipe_bot.{name}")
+
+# Default logger setup
+def setup_default_logging():
+    """Setup default logging configuration"""
+    return setup_logging(
+        log_level="INFO",
+        log_file="./logs/recipe_bot.log",
+        enable_console=True
+    )

backend/config/settings.py

@@ -0,0 +1,176 @@
+# Configuration settings for the Recipe Recommendation Bot
+import os
+from typing import Optional, List
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+class Settings:
+    """Simple settings class that reads environment variables directly"""
+    
+    def __init__(self):
+        # ===========================================
+        # Server Configuration
+        # ===========================================
+        self.PORT = int(os.getenv("PORT", 8000))
+        self.HOST = os.getenv("HOST", "0.0.0.0")
+        self.ENVIRONMENT = os.getenv("ENVIRONMENT", "development")
+        self.DEBUG = os.getenv("DEBUG", "true").lower() == "true"
+        
+        # ===========================================
+        # CORS Configuration
+        # ===========================================
+        cors_origins = os.getenv("CORS_ORIGINS", '["http://localhost:3000","http://localhost:5173","http://localhost:8080"]')
+        self.CORS_ORIGINS = self._parse_list(cors_origins)
+        self.CORS_ALLOW_CREDENTIALS = os.getenv("CORS_ALLOW_CREDENTIALS", "true").lower() == "true"
+        
+        cors_methods = os.getenv("CORS_ALLOW_METHODS", '["GET","POST","PUT","DELETE","OPTIONS"]')
+        self.CORS_ALLOW_METHODS = self._parse_list(cors_methods)
+        
+        cors_headers = os.getenv("CORS_ALLOW_HEADERS", '["*"]')
+        self.CORS_ALLOW_HEADERS = self._parse_list(cors_headers)
+        
+        # ===========================================
+        # LLM & Embedding Provider Configuration
+        # ===========================================
+        self.LLM_PROVIDER = os.getenv("LLM_PROVIDER", "google")
+        self.EMBEDDING_PROVIDER = os.getenv("EMBEDDING_PROVIDER", self.LLM_PROVIDER)  # Default to same as LLM
+        
+        # OpenAI Configuration
+        self.OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+        self.OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-5-nano")
+        self.OPENAI_TEMPERATURE = float(os.getenv("OPENAI_TEMPERATURE", "0.7"))
+        self.OPENAI_MAX_TOKENS = int(os.getenv("OPENAI_MAX_TOKENS", "1000"))
+        
+        # Google AI Configuration
+        self.GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+        self.GOOGLE_MODEL = os.getenv("GOOGLE_MODEL", "gemini-2.5-flash")
+        self.GOOGLE_TEMPERATURE = float(os.getenv("GOOGLE_TEMPERATURE", "0.7"))
+        self.GOOGLE_MAX_TOKENS = int(os.getenv("GOOGLE_MAX_TOKENS", "1000"))
+        
+        # Hugging Face Configuration
+        self.HUGGINGFACE_API_TOKEN = os.getenv("HUGGINGFACE_API_TOKEN")
+        self.HUGGINGFACE_MODEL = os.getenv("HUGGINGFACE_MODEL", "microsoft/DialoGPT-medium")
+        self.HUGGINGFACE_API_URL = os.getenv("HUGGINGFACE_API_URL", "https://api-inference.huggingface.co/models/")
+        self.HUGGINGFACE_USE_GPU = os.getenv("HUGGINGFACE_USE_GPU", "false").lower() == "true"
+        self.HUGGINGFACE_USE_API = os.getenv("HUGGINGFACE_USE_API", "false").lower() == "true"
+        
+        # Ollama Configuration
+        self.OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
+        self.OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3.1:8b")
+        self.OLLAMA_TEMPERATURE = float(os.getenv("OLLAMA_TEMPERATURE", "0.7"))
+        
+        # ===========================================
+        # Embedding Model Configuration 
+        # ===========================================
+        # Note: Embedding provider is determined by LLM_PROVIDER setting above
+        
+        # OpenAI Embeddings
+        self.OPENAI_EMBEDDING_MODEL = os.getenv("OPENAI_EMBEDDING_MODEL", "text-embedding-ada-002")
+        
+        # Google Embeddings
+        self.GOOGLE_EMBEDDING_MODEL = os.getenv("GOOGLE_EMBEDDING_MODEL", "models/embedding-001")
+        
+        # Hugging Face Embeddings
+        self.HUGGINGFACE_EMBEDDING_MODEL = os.getenv("HUGGINGFACE_EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
+        
+        # Ollama Embeddings
+        self.OLLAMA_EMBEDDING_MODEL = os.getenv("OLLAMA_EMBEDDING_MODEL", "nomic-embed-text")
+        
+        # ===========================================
+        # Logging Configuration
+        # ===========================================
+        self.LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
+        self.LOG_FORMAT = os.getenv("LOG_FORMAT", "%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        self.LOG_FILE = os.getenv("LOG_FILE", "./logs/app.log")
+
+        # ===========================================
+        # Langchain Debugging Configuration
+        # ===========================================
+        # Note: set to "true" to enable detailed Langchain logs
+        self.LANGCHAIN_DEBUG = os.getenv("LANGCHAIN_DEBUG", "false").lower() == "true"
+    
+    def _parse_list(self, value: str) -> List[str]:
+        """Parse a string representation of a list into an actual list"""
+        try:
+            # Remove brackets and quotes, split by comma
+            if value.startswith('[') and value.endswith(']'):
+                value = value[1:-1]
+            items = [item.strip().strip('"').strip("'") for item in value.split(',')]
+            return [item for item in items if item]  # Remove empty items
+        except:
+            return ["*"]  # Fallback to allow all
+
+    def get_llm_config(self):
+        """Get LLM configuration based on selected provider"""
+        if self.LLM_PROVIDER == "openai":
+            return {
+                "provider": "openai",
+                "api_key": self.OPENAI_API_KEY,
+                "model": self.OPENAI_MODEL,
+                "temperature": self.OPENAI_TEMPERATURE,
+                "max_tokens": self.OPENAI_MAX_TOKENS
+            }
+        elif self.LLM_PROVIDER == "google":
+            return {
+                "provider": "google",
+                "api_key": self.GOOGLE_API_KEY,
+                "model": self.GOOGLE_MODEL,
+                "temperature": self.GOOGLE_TEMPERATURE,
+                "max_tokens": self.GOOGLE_MAX_TOKENS
+            }
+        elif self.LLM_PROVIDER == "huggingface":
+            return {
+                "provider": "huggingface",
+                "api_token": self.HUGGINGFACE_API_TOKEN,
+                "model": self.HUGGINGFACE_MODEL,
+                "api_url": self.HUGGINGFACE_API_URL,
+                "use_gpu": self.HUGGINGFACE_USE_GPU,
+                "use_api": self.HUGGINGFACE_USE_API
+            }
+        elif self.LLM_PROVIDER == "ollama":
+            return {
+                "provider": "ollama",
+                "base_url": self.OLLAMA_BASE_URL,
+                "model": self.OLLAMA_MODEL,
+                "temperature": self.OLLAMA_TEMPERATURE
+            }
+        else:
+            raise ValueError(f"Unsupported LLM provider: {self.LLM_PROVIDER}")
+    
+    def get_embedding_config(self):
+        """Get embedding configuration based on EMBEDDING_PROVIDER setting"""
+        provider = self.EMBEDDING_PROVIDER
+        
+        if provider == "openai":
+            return {
+                "provider": "openai",
+                "api_key": self.OPENAI_API_KEY,
+                "model": self.OPENAI_EMBEDDING_MODEL
+            }
+        elif provider == "google":
+            return {
+                "provider": "google",
+                "api_key": self.GOOGLE_API_KEY,
+                "model": self.GOOGLE_EMBEDDING_MODEL
+            }
+        elif provider == "huggingface":
+            return {
+                "provider": "huggingface",
+                "model": self.HUGGINGFACE_EMBEDDING_MODEL
+            }
+        elif provider == "ollama":
+            return {
+                "provider": "ollama",
+                "base_url": self.OLLAMA_BASE_URL,
+                "model": self.OLLAMA_EMBEDDING_MODEL
+            }
+        else:
+            raise ValueError(f"Unsupported provider: {provider}. Supported providers: openai, google, huggingface, ollama")
+
+# Create global settings instance
+settings = Settings()
+
+# Note: Vector store and database configuration is in database.py
+# from config.database import db_settings

backend/core/exceptions.py

@@ -0,0 +1,15 @@
+# Custom exception handlers for the Recipe Recommendation Bot
+
+class RecipeNotFoundError(Exception):
+    """Raised when a recipe is not found"""
+    pass
+
+class LLMServiceError(Exception):
+    """Raised when LLM service encounters an error"""
+    pass
+
+class VectorStoreError(Exception):
+    """Raised when vector store operations fail"""
+    pass
+
+# TODO: Add more specific exception classes and error handlers

backend/data/sample_recipes.json

@@ -0,0 +1,138 @@
+[
+  {
+    "title": "Mixed Seafood Coconut Fried Rice",
+    "ingredients": [
+      "jasmine rice",
+      "cooked shrimp", 
+      "prawns",
+      "scallops",
+      "coconut milk",
+      "fish sauce",
+      "soy sauce",
+      "garlic",
+      "onion",
+      "ginger",
+      "green onions",
+      "cilantro",
+      "lime",
+      "vegetable oil",
+      "salt",
+      "pepper"
+    ],
+    "instructions": "1. Heat vegetable oil in large pan over medium-high heat. 2. Add garlic, onion, and ginger, stir-fry until fragrant (about 1 minute). 3. Add cooked jasmine rice and mix well, breaking up any clumps. 4. Add shrimp, prawns, and scallops, cook until heated through (3-4 minutes). 5. Pour in coconut milk and season with fish sauce and soy sauce. 6. Stir everything together and cook for 2-3 minutes until well combined. 7. Garnish with chopped green onions and fresh cilantro. 8. Serve immediately with lime wedges on the side.",
+    "metadata": {
+      "cook_time": "25 minutes",
+      "difficulty": "medium",
+      "servings": "4",
+      "category": "seafood",
+      "image_url": "https://example.com/images/mixed-seafood-coconut-fried-rice.jpg"
+    }
+  },
+  {
+    "title": "Classic Chicken Alfredo Pasta",
+    "ingredients": [
+      "fettuccine pasta",
+      "chicken breast",
+      "heavy cream",
+      "parmesan cheese",
+      "butter",
+      "garlic",
+      "italian seasoning",
+      "salt",
+      "pepper",
+      "olive oil",
+      "parsley"
+    ],
+    "instructions": "1. Cook fettuccine pasta according to package directions, drain and set aside. 2. Season chicken breast with salt, pepper, and Italian seasoning. 3. Heat olive oil in large skillet over medium-high heat. 4. Cook chicken breast until golden brown and cooked through (6-7 minutes per side). 5. Remove chicken and slice into strips. 6. In same skillet, melt butter and add minced garlic, cook for 1 minute. 7. Add heavy cream and bring to gentle simmer. 8. Stir in grated Parmesan cheese until melted and smooth. 9. Add cooked pasta and chicken strips to sauce. 10. Toss everything together and garnish with fresh parsley.",
+    "metadata": {
+      "cook_time": "30 minutes",
+      "difficulty": "easy",
+      "servings": "4",
+      "category": "pasta",
+      "image_url": "https://example.com/images/chicken-alfredo-pasta.jpg"
+    }
+  },
+  {
+    "title": "Vegetarian Black Bean Tacos",
+    "ingredients": [
+      "black beans",
+      "canned corn",
+      "tortillas",
+      "avocado",
+      "lime",
+      "red onion",
+      "cilantro",
+      "cumin",
+      "chili powder",
+      "garlic powder",
+      "salt",
+      "pepper",
+      "olive oil",
+      "lettuce",
+      "tomato",
+      "mexican cheese blend"
+    ],
+    "instructions": "1. Drain and rinse black beans, drain corn. 2. Heat olive oil in skillet over medium heat. 3. Add black beans, corn, cumin, chili powder, garlic powder, salt, and pepper. 4. Cook for 5-7 minutes until heated through and flavors meld. 5. Warm tortillas in dry skillet or microwave. 6. Dice avocado, red onion, and tomato. 7. Squeeze lime juice over diced avocado to prevent browning. 8. Assemble tacos with bean mixture, lettuce, tomato, avocado, onion, and cheese. 9. Garnish with fresh cilantro and serve with lime wedges.",
+    "metadata": {
+      "cook_time": "15 minutes",
+      "difficulty": "easy",
+      "servings": "3",
+      "category": "vegetarian",
+      "image_url": "https://example.com/images/black-bean-tacos.jpg"
+    }
+  },
+  {
+    "title": "Beef and Vegetable Stir Fry",
+    "ingredients": [
+      "beef sirloin",
+      "soy sauce",
+      "sesame oil",
+      "cornstarch",
+      "broccoli",
+      "bell peppers",
+      "carrots",
+      "snap peas",
+      "garlic",
+      "ginger",
+      "vegetable oil",
+      "rice vinegar",
+      "brown sugar",
+      "green onions",
+      "sesame seeds"
+    ],
+    "instructions": "1. Slice beef sirloin into thin strips and marinate with soy sauce, sesame oil, and cornstarch for 15 minutes. 2. Cut broccoli into florets, slice bell peppers and carrots. 3. Heat vegetable oil in large wok or skillet over high heat. 4. Add marinated beef and stir-fry until browned (3-4 minutes). 5. Remove beef and set aside. 6. Add more oil if needed, then add garlic and ginger, stir for 30 seconds. 7. Add hard vegetables (carrots, broccoli) first, stir-fry for 2 minutes. 8. Add bell peppers and snap peas, stir-fry for another 2 minutes. 9. Return beef to pan, add rice vinegar and brown sugar. 10. Stir everything together for 1-2 minutes. 11. Garnish with green onions and sesame seeds.",
+    "metadata": {
+      "cook_time": "20 minutes",
+      "difficulty": "medium",
+      "servings": "4",
+      "category": "beef",
+      "image_url": "https://example.com/images/beef-vegetable-stir-fry.jpg"
+    }
+  },
+  {
+    "title": "Mediterranean Quinoa Salad",
+    "ingredients": [
+      "quinoa",
+      "cucumber",
+      "cherry tomatoes",
+      "red onion",
+      "kalamata olives",
+      "feta cheese",
+      "olive oil",
+      "lemon juice",
+      "oregano",
+      "salt",
+      "pepper",
+      "fresh mint",
+      "parsley"
+    ],
+    "instructions": "1. Rinse quinoa thoroughly and cook according to package directions (usually 1:2 ratio with water). 2. Let cooked quinoa cool completely. 3. Dice cucumber, halve cherry tomatoes, and thinly slice red onion. 4. Pit and halve kalamata olives, crumble feta cheese. 5. In large bowl, combine cooled quinoa with prepared vegetables and olives. 6. Make dressing by whisking together olive oil, lemon juice, oregano, salt, and pepper. 7. Pour dressing over salad and toss well. 8. Add crumbled feta cheese and chopped fresh mint and parsley. 9. Toss gently and let sit for 15 minutes to allow flavors to meld. 10. Serve chilled or at room temperature.",
+    "metadata": {
+      "cook_time": "25 minutes",
+      "difficulty": "easy",
+      "servings": "6",
+      "category": "salad",
+      "image_url": "https://example.com/images/mediterranean-quinoa-salad.jpg"
+    }
+  }
+]

backend/data_minning/__init__.py

@@ -0,0 +1 @@
+# Services package initialization

backend/data_minning/all_nigerian_recipe_scraper.py

@@ -0,0 +1,193 @@
+import re
+from typing import Iterable, List, Optional
+from urllib.parse import urljoin, urlparse
+
+from bs4 import BeautifulSoup, NavigableString, Tag
+from .dto.recipe_doc import RecipeDoc
+from .base_scrapper import BaseRecipeScraper
+from backend.utils.sanitization import clean
+
+
+class AllNigerianRecipesScraper(BaseRecipeScraper):
+    # two-level paths like /beans/beans-porridge/
+    ALLOWED_CATS = {
+        "beans","soups","stews","salad","breakfast","rice",
+        "yam","plantain","swallow","drinks","desserts","meat","fish", "chicken"
+    }
+    PAT_ING   = re.compile(r"\bingredients?\b|\bwhat you need\b|\bingredients? for\b", re.I)
+    PAT_NOTEI = re.compile(r"\bnotes?\b.*ingredients?\b", re.I)
+    PAT_BEFORE= re.compile(r"\bbefore you (cook|grill|fry|bake|roast|steam|boil|prepare)\b", re.I)
+    PAT_INSTR = re.compile(r"\b(preparation|directions|method|instructions|cooking|making)\b", re.I)
+# Map first path segment (category) -> course
+    COURSE_BY_CATEGORY = {
+        "soups": "Soup",
+        "stews": "Stew",
+        "snacks": "Snack",
+        "drinks": "Drink",
+        "desserts": "Dessert",
+        "breakfast": "Breakfast",
+        "salad": "Salad",
+        "rice": "Main Course",
+        "beans": "Main Course",
+        "yam": "Main Course",
+        "plantain": "Main Course",
+        "meat": "Main Course",
+        "fish": "Main Course",
+        "chicken": "Main Course",   # <-- add chicken
+        "swallow": "Side Dish",
+    }
+    # Fallback keyword hints from URL/title if category isn’t enough
+    COURSE_BY_KEYWORD = [
+        (r"\bsoup(s)?\b", "Soup"),
+        (r"\bstew(s)?\b", "Stew"),
+        (r"\bsalad(s)?\b", "Salad"),
+        (r"\b(snack|small[-\s]?chops)\b", "Snack"),
+        (r"\b(drink|juice|smoothie)\b", "Drink"),
+        (r"\bbreakfast\b", "Breakfast"),
+        (r"\bdessert(s)?\b", "Dessert"),
+    ]
+
+    def __init__(self, base_domain="www.allnigerianrecipes.com", index_url="https://www.allnigerianrecipes.com/other/sitemap/"):
+        super().__init__(base_domain)
+        self.index_url = index_url
+
+    def _is_two_level_recipe(self, url: str) -> bool:
+        sp = urlparse(url); segs = [s for s in sp.path.strip("/").split("/") if s]
+        if len(segs) != 2: return False
+        if segs[0].lower() not in self.ALLOWED_CATS: return False
+        bad = {"page","tag","tags","category","categories","author","search"}
+        if any(s in bad for s in segs): return False
+        if sp.path.lower().endswith((".xml",".pdf",".zip",".jpg",".jpeg",".png",".webp",".mp4",".mov")):
+            return False
+        return True
+
+    def discover_urls(self) -> Iterable[str]:
+        soup = self.fetch_soup(self.index_url)
+        seen = set()
+        for a in soup.select("a[href]"):
+            u = urljoin(self.index_url, a["href"])
+            if self.same_domain(u) and self._is_two_level_recipe(u) and u not in seen:
+                seen.add(u); yield u
+
+    def _li_text_only(self, li: Tag) -> str:
+        parts=[]
+        for ch in li.contents:
+            if isinstance(ch, NavigableString): parts.append(str(ch))
+            elif isinstance(ch, Tag) and ch.name not in ("ul","ol","iframe"):
+                parts.append(ch.get_text(" ", strip=True))
+        return clean(" ".join(parts)) or ""
+
+    def _collect_after(self, h: Tag, categories = None) -> List[str]:
+        lvl = int(h.name[1])
+        out = []
+        for sib in h.next_siblings:
+            if isinstance(sib, Tag) and sib.name in self.HEADING_TAGS and int(sib.name[1]) <= lvl:
+                break
+            if isinstance(sib, Tag):
+                if categories == 'ingredients':
+                    if sib.name in ("ul", "ol"):
+                        lis = sib.find_all("li", recursive=False) or sib.find_all("li")
+                        for li in lis:
+                            t = self._li_text_only(li)
+                            if t:
+                                out.append(t)
+                    # Skip other tags for ingredients
+                else:
+                    if sib.name in ("ul", "ol"):
+                        lis = sib.find_all("li", recursive=False) or sib.find_all("li")
+                        for li in lis:
+                            t = self._li_text_only(li)
+                            if t:
+                                out.append(t)
+                    elif sib.name in ("p", "div", "blockquote"):
+                        t = clean(sib.get_text(" ", strip=True))
+                        if t:
+                            out.append(t)
+            elif isinstance(sib, NavigableString):
+                if categories != 'ingredients':  # Only add NavigableString if not ingredients
+                    t = clean(str(sib))
+                    if t:
+                        out.append(t)
+        return [x for x in out if not x.lower().startswith("video of ")]
+
+    def extract_recipe(self, soup: BeautifulSoup, url: str, category: Optional[str] = None) -> RecipeDoc:
+        doc = RecipeDoc.make(url)
+        # JSON-LD first
+        j = self.extract_jsonld(soup)
+        if j:
+            for k,v in j.items():
+                if hasattr(doc, k): setattr(doc, k, v)
+        # Title fallback
+        if not doc.title:
+            title = soup.find("h1") or soup.find("title")
+            doc.title = clean(title.get_text()) if title else None
+
+        root = soup.select_one(".entry-content") or soup
+        sections_ing, sections_instr, notes = [], [], []
+
+        for h in root.find_all(self.HEADING_TAGS):
+            ht = h.get_text(" ", strip=True) or ""
+            if self.PAT_NOTEI.search(ht):
+                it = self._collect_after(h); 
+                if it: notes.append({"title": ht, "items": it})
+            elif self.PAT_ING.search(ht):
+                it = [x for x in self._collect_after(h, 'ingredients') if self._looks_like_ingredient(x)]
+                if it: sections_ing.append({"title": ht, "items": it})
+            elif self.PAT_BEFORE.search(ht) or self.PAT_INSTR.search(ht):
+                st = self._collect_after(h); 
+                if st: sections_instr.append({"title": ht, "steps": st})
+
+        # Flatten
+        for s in sections_ing + notes:
+            doc.ingredients.extend(s["items"])
+        for s in sections_instr:
+            doc.instructions.extend(s["steps"])
+
+        ingredients_text  = self._to_ingredients_text(doc.ingredients)
+        instructions_text = self._to_instructions_text(doc.instructions)
+
+        # Store as text (team requirement)
+        doc.ingredients  = ingredients_text
+        doc.instructions = instructions_text
+        # Notes & image
+        if notes:
+            # concatenate notes into a single field
+            doc.notes = " ".join(["; ".join(n.get("items", [])) for n in notes if n.get("items")])
+        if not doc.image_url:
+            doc.image_url = clean(self.get_meta_image(soup))
+
+        # Infer category (first path segment) and map to a course
+        if not getattr(doc, "category", None) or not getattr(doc, "course", None):
+            cat, crs = self._infer_category_and_course(url, doc.title)
+            if not doc.category:
+                doc.category = cat
+            if not doc.course:
+                doc.course = crs
+
+        return doc
+
+    def _looks_like_ingredient(self, text: str) -> bool:
+        if len(text.split()) > 15: return False
+        # Match numbers followed by units (with or without space), or common ingredient keywords
+        return bool(re.search(r"\b(\d+\s?(cup|cups|tsp|tbsp|teaspoon|tablespoon|g|kg|ml|l|gram)?|cup|cups|tsp|tbsp|teaspoon|tablespoon|g|kg|ml|l|gram|salt|pepper|onion|oil|water|rice|groundnuts|tamarind)\b", text, re.I))
+
+    @staticmethod
+    def _slug_to_title(slug: Optional[str]) -> Optional[str]:
+        return slug.replace("-", " ").title() if slug else None
+
+    def _infer_category_and_course(self, url: str, title: Optional[str]) -> tuple[Optional[str], Optional[str]]:
+        sp = urlparse(url)
+        segs = [s for s in sp.path.strip("/").split("/") if s]
+        cat_slug = segs[0] if segs else None
+        category = self._slug_to_title(cat_slug) if cat_slug else None
+
+        course = self.COURSE_BY_CATEGORY.get(cat_slug)
+        if not course:
+            hay = f"{url} {(title or '')}".lower()
+            for pat, crs in self.COURSE_BY_KEYWORD:
+                if re.search(pat, hay):
+                    course = crs
+                    break
+        if not course and category:
+            course = "Main Course"
+        return category, course

backend/data_minning/base_scrapper.py

@@ -0,0 +1,348 @@
+from dataclasses import asdict
+from datetime import datetime
+import io
+import json
+import os
+import time
+from typing import Any, Callable, Dict, Iterable, List, Optional, Union
+from pathlib import Path
+from click import Tuple
+from pymongo import MongoClient, UpdateOne, errors
+
+from .dto.stream_opts import StreamOptions
+
+from .dto.recipe_doc import RecipeDoc
+
+from .soup_client import SoupClient
+from backend.utils.sanitization import clean
+from bs4 import BeautifulSoup
+from backend.config.database import db_settings
+
+class JsonArraySink:
+    """
+    Append-safe JSON array writer.
+    - Creates file with `[` ... `]`
+    - If file exists, removes trailing `]`, appends items, and re-closes.
+    """
+    def __init__(self, path: str):
+        self.path = path
+        self._opened = False
+        self._first = True
+        self.f = None
+
+    def _prepare(self):
+        if self._opened:
+            return
+
+        # Ensure file exists with an empty array
+        if not os.path.exists(self.path):
+            with open(self.path, "w", encoding="utf-8") as f:
+                f.write("[\n]")
+
+        self.f = open(self.path, "r+", encoding="utf-8")
+
+        # Find the position of the final ']' from the end
+        self.f.seek(0, io.SEEK_END)
+        end = self.f.tell()
+        step = min(4096, end)
+        pos = end
+        last_bracket = -1
+        while pos > 0:
+            pos = max(0, pos - step)
+            self.f.seek(pos)
+            chunk = self.f.read(step)
+            j = chunk.rfind("]")
+            if j != -1:
+                last_bracket = pos + j
+                break
+
+        if last_bracket == -1:
+            # Corrupt file: reset to empty array
+            self.f.seek(0); self.f.truncate(0); self.f.write("[\n]"); self.f.flush()
+            last_bracket = 2  # index of ']' in "[\n]"
+
+        # Decide "is first item?" by inspecting the content BEFORE the ']'
+        self.f.seek(0)
+        prefix = self.f.read(last_bracket).strip()   # content up to (but not including) ']'
+        # Empty array has only '[' (possibly with whitespace/newline)
+        self._first = (prefix == "[")
+
+        # Now remove the closing ']' so we can append
+        self.f.seek(last_bracket)
+        self.f.truncate()
+
+        self._opened = True
+
+    def write_many(self, docs: List[Dict[str, Any]]):
+        if not docs:
+            return
+        self._prepare()
+
+        for d in docs:
+            if not self._first:
+                self.f.write(",\n")
+            else:
+                # First item: no leading comma
+                self._first = False
+            self.f.write(json.dumps(d, ensure_ascii=False, indent=2, default=str))
+
+        # Restore the closing bracket
+        self.f.write("\n]")
+        self.f.flush()
+
+    def close(self):
+        if self.f:
+            self.f.close()
+        self._opened = False
+
+class MongoSink:
+    def __init__(self, ):
+        db_config = db_settings.get_vector_store_config()
+        self.client = MongoClient(db_config["uri"], retryWrites=True, serverSelectionTimeoutMS=10000)
+        self.col = self.client[db_config["database"]][db_config["collection_name"]]
+        self._ensure_indexes()
+
+    def _ensure_indexes(self):
+        self.col.create_index("url", unique=True)
+        self.col.create_index("title")
+        self.col.create_index("category")
+        self.col.create_index("scraped_at")
+
+    def write_many(self, docs: List[Dict[str, Any]]):
+        if not docs: return
+        ops = []
+        now = datetime.utcnow()
+        for d in docs:
+            d = d.copy()
+            d.setdefault("scraped_at", now)
+            ops.append(UpdateOne({"url": d["url"]}, {"$set": d, "$setOnInsert": {"created_at": now}}, upsert=True))
+        try:
+            self.col.bulk_write(ops, ordered=False)
+        except errors.BulkWriteError as e:
+            # duplicates or minor issues won't halt unordered bulk
+            pass
+
+    def close(self):
+        self.client.close()
+
+    # we need to create a search function to fetch recipes by title or ingredients from the embeddings given the embedding fields, db and collection 
+
+class DualSink:
+    def __init__(self, json_sink: Optional[JsonArraySink], mongo_sink: Optional[MongoSink]):
+        self.json = json_sink
+        self.mongo = mongo_sink
+    def write_many(self, docs: List[Dict[str, Any]]):
+        if self.json: self.json.write_many(docs)
+        if self.mongo: self.mongo.upsert_batch(docs)
+    def close(self):
+        if self.json: self.json.close()
+        if self.mongo: self.mongo.close()
+
+
+class BaseRecipeScraper(SoupClient):
+    HEADING_TAGS = ("h1","h2","h3","h4","h5","h6")
+    
+    def __init__(
+        self,
+        *args,
+        embedder= None,
+        embedding_fields= None,
+        **kwargs
+    ):
+        """
+        embedder: HFEmbedder(), optional
+        embedding_fields: list of (source_field, target_field) like:
+            [("title", "title_emb"), ("instructions_text", "instr_emb")]
+        """
+        super().__init__(*args, **kwargs)
+        self.embedder = embedder
+        self.embedding_fields = embedding_fields or []
+        self.logger = self.log
+
+    def extract_jsonld(self, soup: BeautifulSoup) -> Optional[Dict[str, Any]]:
+        def to_list(x): return x if isinstance(x, list) else [x]
+        for tag in soup.find_all("script", type="application/ld+json"):
+            try:
+                data = json.loads(tag.string or "{}")
+            except Exception:
+                continue
+            nodes = (data.get("@graph", [data]) if isinstance(data, dict)
+                     else (data if isinstance(data, list) else []))
+            for n in nodes:
+                if not isinstance(n, dict): continue
+                t = n.get("@type")
+                if t == "Recipe" or (isinstance(t, list) and "Recipe" in t):
+                    doc = RecipeDoc()
+                    doc.title = clean(n.get("name"))
+                    # ingredients
+                    ings = []
+                    for ing in to_list(n.get("recipeIngredient") or []):
+                        if isinstance(ing, dict):
+                            ings.append(clean(ing.get("name") or ing.get("text")))
+                        else:
+                            ings.append(clean(str(ing)))
+                    doc.ingredients = [x for x in ings if x]
+                    # instructions
+                    steps = []
+                    for st in to_list(n.get("recipeInstructions") or []):
+                        if isinstance(st, dict):
+                            steps.append(clean(st.get("text") or st.get("name")))
+                        else:
+                            steps.append(clean(str(st)))
+                    doc.instructions = [x for x in steps if x]
+                   
+                    doc.servings = n.get("recipeYield")
+                    doc.image_url = clean((n.get("image") or {}).get("url") if isinstance(n.get("image"), dict) else (n.get("image")[0] if isinstance(n.get("image"), list) else n.get("image")))
+                    doc.course = clean(n.get("recipeCategory")) if isinstance(n.get("recipeCategory"), str) else None
+                    doc.cuisine = clean(n.get("recipeCuisine")) if isinstance(n.get("recipeCuisine"), str) else None
+                    return asdict(doc)
+        return None
+    
+    @staticmethod
+    def _dedupe_preserve_order(items: List[str]) -> List[str]:
+        seen = set()
+        out = []
+        for x in items:
+            x = clean(x)
+            if not x or x in seen: 
+                continue
+            seen.add(x); out.append(x)
+        return out
+
+    @staticmethod
+    def _to_ingredients_text(items: List[str]) -> str:
+        """
+        Turn ingredient bullets into a single text block.
+        Using one-per-line is great for embeddings and human readability.
+        """
+        items = [clean(x) for x in items if x]
+        items = BaseRecipeScraper._dedupe_preserve_order(items)
+        return "\n".join(f"- {x}" for x in items)
+
+    @staticmethod
+    def _to_instructions_text(steps: List[str]) -> str:
+        """
+        Turn ordered steps into a single text block.
+        Numbered paragraphs help embeddings keep sequence context.
+        """
+        steps = [clean(x) for x in steps if x]
+        steps = BaseRecipeScraper._dedupe_preserve_order(steps)
+        return "\n\n".join(f"{i}. {s}" for i, s in enumerate(steps, 1))
+    # site-specific scrapers override these two:
+    def discover_urls(self) -> Iterable[str]:
+        raise NotImplementedError
+    def extract_recipe(self, soup: BeautifulSoup, url: str, category: Optional[str] = None) -> RecipeDoc:
+        raise NotImplementedError
+
+    # shared streaming loop
+    def stream(self, sink: DualSink, options: Optional[StreamOptions] = None) -> int:
+        opts = options or StreamOptions()
+        self.log.info(
+            f"Starting stream: limit={opts.limit} batch_size={opts.batch_size} "
+            f"resume_file={opts.resume_file} sink={type(sink).__name__}"
+        )
+
+        processed = set()
+        if opts.resume_file:
+            resume_path = Path("data") / opts.resume_file  # <-- not ../data
+            print(resume_path, 'resume_path')
+            if resume_path.exists():                       # <-- open only if it exists
+                with resume_path.open("r", encoding="utf-8") as f:
+                    processed = {line.strip() for line in f if line.strip()}
+            else:
+                processed = set()
+            self.log.info(f"[resume] {len(processed)} URLs already done")
+
+        batch, saved = [], 0
+        try:
+            for i, url in enumerate(self.discover_urls(), 1):
+                if opts.limit and i > opts.limit: break
+                if not self.same_domain(url): continue
+                if url in processed: continue
+
+                try:
+                    soup = self.fetch_soup(url)
+                    doc = self.extract_recipe(soup, url)
+                    doc.finalize()
+                    batch.append(asdict(doc))
+                except Exception as e:
+                    self.log.warning(f"[skip] {url} -> {e}")
+
+                if opts.resume_file:
+                    resume_path = Path("data") / opts.resume_file
+                    with open(resume_path, "a", encoding="utf-8") as rf:
+                        rf.write(url + "\n")
+
+                if len(batch) >= opts.batch_size:
+                    self._apply_embeddings(batch)
+                    sink.write_many(batch); saved += len(batch); batch = []
+                    if opts.progress_callback: opts.progress_callback(saved)
+                    self.log.info(f"[resume] {saved} URLs already done 1")
+
+                if i % 25 == 0:
+                    self.log.info(f"…processed {i}, saved {saved}")
+
+                time.sleep(opts.delay)
+
+            if batch:
+                self._apply_embeddings(batch)
+                sink.write_many(batch); saved += len(batch)
+                if opts.progress_callback: opts.progress_callback(saved)
+                self.log.info(f"[resume] {saved} URLs already done2 ")
+        finally:
+            sink.close()
+
+        self.log.info(f"[done] saved {saved}")
+        return saved
+    
+    @staticmethod
+    def _field_to_text(val: Any) -> str:
+        if isinstance(val, list):
+            return "\n".join(str(x) for x in val)
+        if val is None:
+            return ""
+        return str(val)
+
+    def _gather_text(self, doc: Dict[str, Any], src: Any) -> str:
+        if isinstance(src, tuple):
+            parts: List[str] = []
+            for f in src:
+                t = self._field_to_text(doc.get(f))
+                if t:
+                    # Optional: label sections to help the embedder
+                    label = "Ingredients" if f == "ingredients" else ("Instructions" if f == "instructions" else f)
+                    parts.append(f"{label}:\n{t}")
+            return "\n\n".join(parts)
+        else:
+            return self._field_to_text(doc.get(src))
+        
+    def _apply_embeddings(self, batch: List[Dict[str, Any]]) -> None:
+        """
+        Applies embeddings to specified fields in a batch of documents.
+
+        For each (source_field, destination_field) pair in `self.embedding_fields`, this method:
+        - Extracts the value from `source_field` in each document of the batch.
+        - Converts the value to a string. If the value is a list, joins its elements with newlines.
+        - Handles `None` values by converting them to empty strings.
+        - Uses `self.embedder.encode` to generate embeddings for the processed texts.
+        - Stores the resulting embedding vector in `destination_field` of each document.
+
+        If `self.embedder`, `self.embedding_fields`, or `batch` is not set or empty, the method returns immediately.
+
+        Args:
+            batch (List[Dict[str, Any]]): A list of documents to process, where each document is a dictionary.
+
+        Returns:
+            None
+        """
+        if not self.embedder or not self.embedding_fields or not batch:
+            return
+        try:
+            for src_spec, dst_field in self.embedding_fields:
+                texts = [ self._gather_text(doc, src_spec) for doc in batch ]
+                embs = self.embedder.encode(texts)
+                for document, vec in zip(batch, embs):
+                    document[dst_field] = vec
+        except Exception as e:
+            self.logger.warning(f"[stream error]: {e}")
+

backend/data_minning/dto/__init__.py

@@ -0,0 +1 @@
+# Services package initialization

backend/data_minning/dto/recipe_doc.py

@@ -0,0 +1,45 @@
+from typing import Optional, List, Dict, Any
+from dataclasses import dataclass, asdict, field
+from datetime import datetime
+from urllib.parse import urlparse
+
+@dataclass
+class RecipeDoc:
+    title: Optional[str] = None
+    url: Optional[str] = None
+    source: Optional[str] = None
+    category: Optional[str] = None
+    ingredients: List[str] = field(default_factory=list)
+    instructions: List[str] = field(default_factory=list)
+    prep_time: Optional[str] = None
+    cook_time: Optional[str] = None
+    total_time: Optional[str] = None
+    servings: Optional[Any] = None
+    calories: Optional[float] = None
+    rating: Optional[float] = None
+    rating_count: Optional[int] = None
+    course: Optional[str] = None
+    cuisine: Optional[str] = None
+    notes: Optional[str] = None
+    image_url: Optional[str] = None
+    needs_review: Optional[bool] = None
+    section: List[Dict[str, Any]] = field(default_factory=list)
+    # internal
+    scraped_at: Optional[datetime] = None
+
+    @staticmethod
+    def make(url: str) -> "RecipeDoc":
+        host = urlparse(url).netloc
+        RecipeDocs = RecipeDoc(url=url, source=host, scraped_at=datetime.utcnow())
+        return RecipeDocs
+
+    def finalize(self):
+        # mark needs_review conservatively
+        self.needs_review = bool((len(self.ingredients) < 1) or (len(self.instructions) < 1))
+        # normalize empties to None when appropriate
+        for k, v in list(asdict(self).items()):
+            if isinstance(v, list) and len(v) == 0:
+                setattr(self, k, [] if k in ("ingredients","instructions","section") else None)
+            elif v == "":
+                setattr(self, k, None)
+    

backend/data_minning/dto/stream_opts.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+
+@dataclass
+class StreamOptions:
+    delay: float = 0.3
+    limit: Optional[int] = None
+    batch_size: int = 50
+    resume_file: Optional[str] = None
+    # progress gets total_saved so far
+    progress_callback: Optional[Callable[[int], None]] = None

backend/data_minning/soup_client.py

@@ -0,0 +1,43 @@
+import logging
+from typing import Optional
+from urllib.parse import urlparse
+
+import requests
+from bs4 import BeautifulSoup
+
+class SoupClient:
+    def __init__(self, base_domain: str, user_agent: str = None):
+        self.base_domain = base_domain
+        self.base_url = f"https://{base_domain}"
+        self.session = requests.Session()
+        self.session.headers.update({
+            "User-Agent": user_agent or
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+            "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+        })
+        self.log = logging.getLogger(self.__class__.__name__)
+
+    def same_domain(self, url: str) -> bool:
+        host = urlparse(url).netloc.lower()
+        return host == self.base_domain.lower() or host == self.base_domain.lower().replace("www.", "")
+
+    def fetch_soup(self, url: str, timeout: int = 30) -> BeautifulSoup:
+        r = self.session.get(url, timeout=timeout)
+        r.raise_for_status()
+        return BeautifulSoup(r.content, "lxml")
+
+    def close(self):
+        self.session.close()
+    
+    def get_meta_image(self, soup: BeautifulSoup) -> Optional[str]:
+        for sel in [
+            "meta[property='og:image']",
+            "meta[name='og:image']",
+            "meta[name='twitter:image']",
+            "meta[property='twitter:image']"
+        ]:
+            m = soup.select_one(sel)
+            if m and m.get("content"):
+                return m["content"]
+        img = soup.find("img")
+        return (img.get("src") or img.get("data-src")) if img else None

backend/data_minning/yummy_medley_scraper.py

@@ -0,0 +1,209 @@
+import re
+from typing import Iterable, Optional
+from urllib.parse import urljoin, urlparse
+from bs4 import BeautifulSoup
+from .base_scrapper import BaseRecipeScraper, RecipeDoc
+from backend.utils.sanitization import clean
+
+class YummyMedleyScraper(BaseRecipeScraper):
+    """Specifically targets WPRM (WP Recipe Maker) blocks on yummymedley.com"""
+
+    # Only collect tags from the home page tag-cloud widget
+    TAG_CLOUD_SELECTORS = [
+        "#tag_cloud-4 .tagcloud a[href*='/tag/']",
+        "div.widget_tag_cloud .tagcloud a[href*='/tag/']",
+    ]
+
+    # How we find post links on a tag page (grid cards & headers)
+    POST_LINK_SELECTORS = [
+        "#main ul.sp-grid li article .post-header a[href]",  # header link
+        "#main ul.sp-grid li article .post-img a[href]",     # image link
+        "#main article .post-header a[href]",                # fallback
+    ]
+
+    TAG_RE = re.compile(r"/tag/[^/]+/?$")
+
+    def __init__(self, base_domain="www.yummymedley.com"):
+        super().__init__(base_domain)
+
+    # --- NEW: get tag URLs only from home page tag cloud ---
+    def _discover_tag_urls_from_home(self) -> list[str]:
+        soup = self.fetch_soup(self.base_url)
+        tags = set()
+
+        # prefer strict selector; gracefully fall back
+        anchors = []
+        for sel in self.TAG_CLOUD_SELECTORS:
+            anchors = soup.select(sel)
+            if anchors:
+                break
+
+        if not anchors:
+            # final fallback: any /tag/... link on home page
+            anchors = soup.find_all("a", href=self.TAG_RE)
+
+        for a in anchors:
+            href = a.get("href")
+            if href:
+                tags.add(urljoin(self.base_url, href))
+        return sorted(tags)
+
+    # --- NEW: extract post links from a single tag page soup ---
+    def _extract_post_links_from_tag_page(self, soup: BeautifulSoup) -> set[str]:
+        links = set()
+        for sel in self.POST_LINK_SELECTORS:
+            for a in soup.select(sel):
+                href = a.get("href")
+                if href:
+                    links.add(urljoin(self.base_url, href))
+            if links:
+                break  # got some via this selector
+        return links
+
+    # --- helper: is this URL an article we should open? ---
+
+    # --- REWRITTEN: discover only from home-page tag cloud, then crawl each tag ---
+    def discover_urls(self) -> Iterable[str]:
+        tags = self._discover_tag_urls_from_home()
+        if not tags:
+            # Safety: if tag cloud not found, bail early (or fallback to /recipes/)
+            self.logger.warning("No tags discovered from home page tag cloud; falling back to /recipes/")
+            tags = [urljoin(self.base_url, "/recipes/")]
+
+        seen = set()
+        for tag_url in tags:
+            page = 1
+            while page <= 20:  # hard cap to avoid runaway pagination
+                url = tag_url if page == 1 else f"{tag_url.rstrip('/')}/page/{page}/"
+                try:
+                    soup = self.fetch_soup(url)
+                except Exception as e:
+                    self.logger.warning(f"[tag] fetch failed {url}: {e}")
+                    break
+
+                post_links = self._extract_post_links_from_tag_page(soup)
+                if not post_links:
+                    # no posts found -> stop paginating this tag
+                    break
+
+                for u in sorted(post_links):
+                    if u not in seen and self._looks_like_article(u):
+                        seen.add(u)
+                        yield u
+
+                # pagination: look for 'next' or 'older'
+                next_link = (
+                    soup.find("a", string=re.compile(r"next|older", re.I)) or
+                    soup.find("a", rel="next")
+                )
+                if not next_link:
+                    break
+                page += 1
+
+    def _looks_like_article(self, u: str) -> bool:
+        sp = urlparse(u)
+        if not self.same_domain(u): return False
+        if re.search(r"/(tag|category|author|page|feed)/", sp.path, re.I): return False
+        if sp.path.endswith((".xml",".jpg",".png",".pdf",".webp",".zip")): return False
+        segs = [s for s in sp.path.strip("/").split("/") if s]
+        return 1 <= len(segs) <= 3
+    
+    def extract_recipe(self, soup: BeautifulSoup, url: str, category: Optional[str] = None) -> RecipeDoc:
+        doc = RecipeDoc.make(url)
+        # JSON-LD first (many WPRM pages also embed it)
+        j = self.extract_jsonld(soup)
+        if j:
+            for k, v in j.items():
+                if not hasattr(doc, k):
+                    continue
+                # skip empty values from JSON-LD
+                if v in (None, "", [], {}):
+                    continue
+                # never overwrite an already-set url/source
+                if k in ("url", "source") and getattr(doc, k):
+                    continue
+                setattr(doc, k, v)
+        # WPRM block
+        w = soup.find("div", class_="wprm-recipe-container")
+        if w:
+            if not doc.title:
+                t = w.find(class_="wprm-recipe-name")
+                doc.title = clean(t.get_text()) if t else doc.title
+            # image
+            if not doc.image_url:
+                img = w.find("img")
+                doc.image_url = clean(img.get("src") or img.get("data-src")) if img else clean(self.get_meta_image(soup))
+            # rating
+            r = w.find(class_="wprm-recipe-rating-average")
+            if r:
+                try: doc.rating = float(r.get_text().strip())
+                except: pass
+            rc = w.find(class_="wprm-recipe-rating-count")
+            if rc:
+                try: doc.rating_count = int(rc.get_text().strip())
+                except: pass
+            # times
+            def pick(c): 
+                x = w.find(class_=c)
+                return clean(x.get_text()) if x else None
+            doc.prep_time = pick("wprm-recipe-prep_time") or doc.prep_time
+            doc.cook_time = pick("wprm-recipe-cook_time") or doc.cook_time
+            doc.total_time= pick("wprm-recipe-total_time") or doc.total_time
+            # servings
+            s = w.find(class_="wprm-recipe-servings")
+            if s:
+                txt = s.get_text().strip()
+                doc.servings = int(txt) if txt.isdigit() else clean(txt)
+            # calories
+            cal = w.find(class_="wprm-recipe-calories")
+            if cal:
+                try: doc.calories = float(cal.get_text().strip())
+                except: pass
+            # course/cuisine
+            cse = w.find(class_="wprm-recipe-course"); doc.course = clean(cse.get_text()) if cse else doc.course
+            cui = w.find(class_="wprm-recipe-cuisine"); doc.cuisine = clean(cui.get_text()) if cui else doc.cuisine
+            # ingredients
+            ings = []
+            ic = w.find(class_="wprm-recipe-ingredients-container")
+            if ic:
+                for ing in ic.find_all(class_="wprm-recipe-ingredient"):
+                    parts=[]
+                    for cls in ("wprm-recipe-ingredient-amount","wprm-recipe-ingredient-unit","wprm-recipe-ingredient-name","wprm-recipe-ingredient-notes"):
+                        el = ing.find(class_=cls)
+                        if el:
+                            t = el.get_text().strip()
+                            if t:
+                                parts.append(t if "notes" not in cls else f"({t})")
+                    txt = clean(" ".join(parts))
+                    if txt: ings.append(txt)
+            if ings: doc.ingredients = ings
+            # instructions
+            steps=[]
+            ic2 = w.find(class_="wprm-recipe-instructions-container")
+            if ic2:
+                for ins in ic2.find_all(class_="wprm-recipe-instruction"):
+                    t = ins.find(class_="wprm-recipe-instruction-text") or ins
+                    txt = clean(t.get_text().strip())
+                    if txt: steps.append(txt)
+            if steps: doc.instructions = steps
+
+        ingredients_text  = self._to_ingredients_text(doc.ingredients)
+        instructions_text = self._to_instructions_text(doc.instructions)
+
+        # Store as text (team requirement)
+        doc.ingredients  = ingredients_text
+        doc.instructions = instructions_text
+
+        # Fallback title & image
+        if not doc.title:
+            h1 = soup.find("h1") or soup.find("title")
+            doc.title = clean(h1.get_text()) if h1 else None
+        if not doc.image_url:
+            doc.image_url = clean(self.get_meta_image(soup))
+
+        # Optional category (yours wants "Afro-tropical Recipes" etc. — grab from breadcrumbs or page tags if available)
+        cat = soup.find("a", href=re.compile(r"/category/|/tag/"))
+        doc.category = clean(cat.get_text()) if cat else doc.category
+
+        return doc
+

backend/docs/README.md

@@ -0,0 +1,85 @@
+# Documentation Index
+
+## 🚨 Troubleshooting (Start Here)
+
+### Quick Fixes
+- **[Embedding Troubleshooting](./embedding-troubleshooting.md)** - Fix dimension mismatch errors in 5 minutes
+- **[Logging Guide](./logging_guide.md)** - Understanding error messages and logs
+
+### Common Error Messages
+| Error | Quick Fix |
+|-------|-----------|
+| `shapes (768,) and (384,) not aligned` | [Embedding dimension mismatch](./embedding-troubleshooting.md#shapes-768-and-384-not-aligned) |
+| `MongoDB connection failed` | Check `MONGODB_URI` in `.env` |
+| `ChromaDB not available` | `pip install langchain_chroma` |
+| `OpenAI API key invalid` | Update `OPENAI_API_KEY` in `.env` |
+
+## 📖 Comprehensive Guides
+
+### Setup & Configuration
+- **[Embedding Compatibility Guide](./embedding-compatibility-guide.md)** - Complete guide to embedding models and dimensions
+- **[Model Configuration Guide](./model-configuration-guide.md)** - Provider-specific settings, temperature limitations, and best practices
+- **[Architecture Documentation](../docs/architecture.md)** - System overview and design patterns
+- **[Deployment Guide](./deployment.md)** - Production deployment instructions
+
+### API & Development
+- **[API Documentation](./api-documentation.md)** - Detailed API reference
+- **[ChromaDB Refresh Guide](./chromadb_refresh.md)** - Database management and refresh procedures
+
+## 🔧 Developer Resources
+
+### Configuration Examples
+```bash
+# Most reliable setup (384D embeddings)
+EMBEDDING_PROVIDER=huggingface
+HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+
+# Local inference setup (768D embeddings)  
+EMBEDDING_PROVIDER=ollama
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text:v1.5
+
+# Premium quality setup (1536D embeddings)
+EMBEDDING_PROVIDER=openai
+OPENAI_EMBEDDING_MODEL=text-embedding-3-small
+```
+
+### Quick Commands
+```bash
+# Check current embedding configuration
+grep EMBEDDING .env
+
+# Test API health
+curl http://localhost:8080/health
+
+# Clear conversation memory
+curl -X POST http://localhost:8080/clear-memory
+
+# View recent logs
+tail -f ./logs/recipe_bot.log
+```
+
+## 📋 File Organization
+
+```
+docs/
+├── README.md                          # This file
+├── embedding-troubleshooting.md       # 🚨 Quick fixes
+├── embedding-compatibility-guide.md   # 📖 Complete embedding guide
+├── model-configuration-guide.md       # ⚙️  LLM provider configurations
+├── logging_guide.md                   # 🔍 Log analysis
+├── chromadb_refresh.md                # 🔄 Database management
+├── api-documentation.md               # 📡 API reference
+├── architecture.md                    # 🏗️ System design
+└── deployment.md                      # 🚀 Production setup
+```
+
+## 🆘 Getting Help
+
+1. **Check error logs**: `tail -f ./logs/recipe_bot.log`
+2. **Common issues**: Start with [Embedding Troubleshooting](./embedding-troubleshooting.md)
+3. **API problems**: See [API Documentation](./api-documentation.md)
+4. **Setup issues**: Review [Embedding Compatibility Guide](./embedding-compatibility-guide.md)
+
+---
+
+💡 **Tip**: Bookmark the [Embedding Troubleshooting](./embedding-troubleshooting.md) page - it solves 80% of common issues!

backend/docs/chromadb_refresh.md

@@ -0,0 +1,228 @@
+# ChromaDB Refresh Feature Documentation
+
+## Overview
+
+The ChromaDB refresh feature allows you to automatically delete and recreate your local vector database on application startup. This is useful when you add new recipe files or update existing content that needs to be re-indexed.
+
+## Configuration
+
+### Environment Variables
+
+Add the following to your `.env` file:
+
+```bash
+# Set to true to delete and recreate DB on startup (useful for adding new recipes)
+DB_REFRESH_ON_START=false
+```
+
+**Default:** `false` (disabled)
+
+### Environment Files Updated
+
+- ✅ `.env` - Your local configuration
+- ✅ `.env.example` - Template for new deployments  
+- ✅ `config/database.py` - Configuration class updated
+- ✅ `services/vector_store.py` - Implementation added
+
+## How It Works
+
+### Normal Operation (DB_REFRESH_ON_START=false)
+1. Check if `DB_PERSIST_DIRECTORY` exists
+2. If exists with data → Load existing ChromaDB
+3. If empty/missing → Create new ChromaDB from recipe files
+
+### Refresh Mode (DB_REFRESH_ON_START=true)
+1. Check if `DB_PERSIST_DIRECTORY` exists  
+2. If exists → **Delete entire directory** 🚨
+3. Create new ChromaDB from recipe files in `./data/recipes/`
+4. All data is re-indexed with current embedding model
+
+## Usage Examples
+
+### Adding New Recipes
+
+```bash
+# 1. Add new recipe files to ./data/recipes/
+cp new_recipes.json ./data/recipes/
+
+# 2. Enable refresh in .env
+DB_REFRESH_ON_START=true
+
+# 3. Start application (will recreate database)
+uvicorn app:app --reload
+
+# 4. Disable refresh (IMPORTANT!)
+DB_REFRESH_ON_START=false
+```
+
+### Changing Embedding Models
+
+```bash
+# 1. Change embedding provider in .env
+EMBEDDING_PROVIDER=openai
+OPENAI_EMBEDDING_MODEL=text-embedding-3-large
+
+# 2. Enable refresh to rebuild vectors
+DB_REFRESH_ON_START=true
+
+# 3. Start application
+uvicorn app:app --reload
+
+# 4. Disable refresh
+DB_REFRESH_ON_START=false
+```
+
+### Troubleshooting Vector Issues
+
+```bash
+# If ChromaDB is corrupted or having issues
+DB_REFRESH_ON_START=true
+# Restart app to rebuild from scratch
+```
+
+## Important Warnings ⚠️
+
+### Data Loss Warning
+- **Refresh DELETES ALL existing vector data**
+- **This operation CANNOT be undone**
+- Always backup important data before refresh
+
+### Performance Impact
+- Re-indexing takes time (depends on recipe count)
+- Embedding API calls cost money (OpenAI, Google)
+- Application startup will be slower during refresh
+
+### Memory Usage
+- Large recipe datasets require more memory during indexing
+- Monitor system resources during refresh
+
+## Best Practices
+
+### ✅ DO
+- Set `DB_REFRESH_ON_START=false` after refresh completes
+- Test refresh in development before production
+- Monitor logs during refresh process
+- Add new recipes in batches if possible
+
+### ❌ DON'T  
+- Leave refresh enabled in production
+- Refresh unnecessarily (wastes resources)
+- Interrupt refresh process (may corrupt data)
+- Forget to disable after refresh
+
+## Monitoring and Logs
+
+The refresh process is fully logged:
+
+```
+🔄 DB_REFRESH_ON_START=true - Deleting existing ChromaDB at ./data/chromadb_persist
+✅ Existing ChromaDB deleted successfully  
+🆕 Creating new ChromaDB at ./data/chromadb_persist
+✅ Created ChromaDB with 150 document chunks
+```
+
+## Configuration Reference
+
+### Complete Environment Setup
+
+```bash
+# Vector Store Configuration
+VECTOR_STORE_PROVIDER=chromadb
+DB_PATH=./data/chromadb
+DB_COLLECTION_NAME=recipes  
+DB_PERSIST_DIRECTORY=./data/chromadb_persist
+
+# Refresh Control
+DB_REFRESH_ON_START=false  # Set to true only when needed
+
+# Embedding Configuration  
+EMBEDDING_PROVIDER=huggingface
+HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+```
+
+### Database Configuration Object
+
+```python
+from config.database import DatabaseSettings
+
+db_settings = DatabaseSettings()
+config = db_settings.get_vector_store_config()
+
+# Access refresh setting
+refresh_enabled = config['refresh_on_start']  # boolean
+```
+
+## Troubleshooting
+
+### Common Issues
+
+**Refresh not working:**
+- Check `.env` file has `DB_REFRESH_ON_START=true`
+- Verify environment is loaded correctly
+- Check file permissions on persist directory
+
+**Application won't start after refresh:**
+- Check recipe files exist in `./data/recipes/`
+- Verify embedding provider credentials
+- Review application logs for specific errors
+
+**Partial refresh/corruption:**
+- Delete persist directory manually
+- Set refresh=true and restart
+- Check disk space availability
+
+### Emergency Recovery
+
+If refresh fails or corrupts data:
+
+```bash
+# Manual cleanup
+rm -rf ./data/chromadb_persist
+
+# Reset configuration  
+DB_REFRESH_ON_START=true
+
+# Restart application
+uvicorn app:app --reload
+```
+
+## Testing
+
+Test the refresh functionality:
+
+```bash
+# Run refresh tests
+python3 test_refresh.py
+
+# Demo the feature
+python3 demo_refresh.py
+```
+
+## Implementation Details
+
+### Files Modified
+
+1. **`config/database.py`**
+   - Added `DB_REFRESH_ON_START` environment variable
+   - Updated `get_vector_store_config()` method
+
+2. **`services/vector_store.py`**  
+   - Added `shutil` import for directory deletion
+   - Implemented refresh logic in `_get_or_create_vector_store()`
+   - Added comprehensive logging
+
+3. **Environment Files**
+   - Updated `.env` and `.env.example` with new variable
+   - Added documentation comments
+
+### Code Changes
+
+```python
+# In vector_store.py
+if refresh_on_start and persist_dir.exists():
+    logger.info(f"🔄 DB_REFRESH_ON_START=true - Deleting existing ChromaDB at {persist_dir}")
+    shutil.rmtree(persist_dir) 
+    logger.info(f"✅ Existing ChromaDB deleted successfully")
+```
+
+This feature provides a simple but powerful way to manage vector database content lifecycle while maintaining data integrity and providing clear user control.

backend/docs/embedding-compatibility-guide.md

@@ -0,0 +1,249 @@
+# Embedding Compatibility Guide
+
+## 🔍 Understanding Embedding Dimensions
+
+When working with vector databases and embeddings, **dimension compatibility** is crucial for successful similarity searches. This guide helps you understand and troubleshoot embedding dimension issues.
+
+## 📊 Common Embedding Models & Their Dimensions
+
+| Provider | Model | Dimensions | Use Case |
+|----------|-------|------------|----------|
+| **HuggingFace** | `sentence-transformers/all-MiniLM-L6-v2` | **384** | Fast, lightweight, good for most tasks |
+| **HuggingFace** | `sentence-transformers/all-mpnet-base-v2` | **768** | Higher quality, larger model |
+| **Ollama** | `nomic-embed-text:v1.5` | **768** | Local inference, privacy-focused |
+| **Ollama** | `mxbai-embed-large` | **1024** | High-quality local embeddings |
+| **OpenAI** | `text-embedding-3-small` | **1536** | Commercial API, good performance |
+| **OpenAI** | `text-embedding-3-large` | **3072** | Highest quality, expensive |
+| **Google** | `models/embedding-001` | **768** | Google AI integration |
+
+## ⚠️ Common Error: Dimension Mismatch
+
+### Symptoms
+```
+WARNING - [custom_mongo_vector.py:103] - ⚠️ Error processing document: shapes (768,) and (384,) not aligned
+```
+
+### Root Cause
+Your **query embeddings** and **stored embeddings** have different dimensions:
+- Query: Generated with Model A (e.g., 768 dimensions)
+- Stored: Created with Model B (e.g., 384 dimensions)
+
+### Why This Happens
+1. You changed embedding models after creating your database
+2. Your database was created with a different embedding provider
+3. Environment configuration doesn't match the original setup
+
+## 🔧 Solution Strategies
+
+### Strategy 1: Match Your Current Database (Recommended)
+
+**Step 1: Identify stored embedding dimensions**
+```bash
+# Check your MongoDB collection to see stored embedding dimensions
+# Look at the 'ingredients_emb' field length
+```
+
+**Step 2: Update .env to match**
+```bash
+# If stored embeddings are 384-dimensional (common with all-MiniLM-L6-v2)
+EMBEDDING_PROVIDER=huggingface
+HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+
+# If stored embeddings are 768-dimensional
+EMBEDDING_PROVIDER=ollama
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text:v1.5
+```
+
+### Strategy 2: Regenerate Database with New Model
+
+**Step 1: Choose your preferred embedding model**
+```bash
+# Example: Use Ollama for local inference
+EMBEDDING_PROVIDER=ollama
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text:v1.5
+```
+
+**Step 2: Enable database refresh**
+```bash
+DB_REFRESH_ON_START=true
+```
+
+**Step 3: Restart application**
+```bash
+uvicorn app:app --reload
+```
+
+**Step 4: Disable refresh (Important!)**
+```bash
+DB_REFRESH_ON_START=false
+```
+
+## 🔍 Debugging Embedding Issues
+
+### Check Current Configuration
+```bash
+# View your current embedding setup
+grep -E "EMBEDDING_PROVIDER|_EMBEDDING_MODEL" .env
+```
+
+### Monitor Embedding Dimensions
+The custom MongoDB vector store now logs dimension information:
+```
+🔢 Query embedding dimensions: 768
+⚠️ Dimension mismatch: query=768D, stored=384D
+💡 Consider changing EMBEDDING_PROVIDER to match stored embeddings
+```
+
+### Verify Database Content
+```python
+# Check stored embedding dimensions in MongoDB
+collection.find_one({"ingredients_emb": {"$exists": True}})["ingredients_emb"]
+# Count the array length to get dimensions
+```
+
+## 📋 Environment Configuration Examples
+
+### Example 1: HuggingFace (384D) - Most Common
+```bash
+# .env configuration for 384-dimensional embeddings
+EMBEDDING_PROVIDER=huggingface
+HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+HUGGINGFACE_API_TOKEN=your_token_here
+```
+
+### Example 2: Ollama (768D) - Local Inference
+```bash
+# .env configuration for 768-dimensional embeddings
+EMBEDDING_PROVIDER=ollama
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text:v1.5
+OLLAMA_BASE_URL=http://localhost:11434
+```
+
+### Example 3: OpenAI (1536D) - Premium Quality
+```bash
+# .env configuration for 1536-dimensional embeddings
+EMBEDDING_PROVIDER=openai
+OPENAI_EMBEDDING_MODEL=text-embedding-3-small
+OPENAI_API_KEY=your_api_key_here
+```
+
+## 🚨 Common Pitfalls
+
+### 1. Mixed Providers
+❌ **Don't do this:**
+```bash
+# Database created with HuggingFace
+EMBEDDING_PROVIDER=huggingface  # Original
+
+# Later changed to Ollama without refreshing DB
+EMBEDDING_PROVIDER=ollama  # New - causes dimension mismatch!
+```
+
+### 2. Forgetting to Disable Refresh
+❌ **Don't forget:**
+```bash
+# After refreshing database, always disable refresh
+DB_REFRESH_ON_START=false  # SET THIS BACK TO FALSE!
+```
+
+### 3. Model Name Typos
+❌ **Watch out for:**
+```bash
+# Typo in model name will cause failures
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text:v1.5  ✅
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text       ❌ (missing version)
+```
+
+## 📊 Performance Comparison
+
+| Model | Speed | Quality | Dimensions | Local/API | Cost |
+|-------|-------|---------|------------|-----------|------|
+| `all-MiniLM-L6-v2` | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | 384 | Both | Free |
+| `nomic-embed-text:v1.5` | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ | 768 | Local | Free |
+| `text-embedding-3-small` | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | 1536 | API | $$$ |
+
+## 🔧 Troubleshooting Steps
+
+### Step 1: Check Current Setup
+```bash
+# 1. Check your environment configuration
+cat .env | grep EMBEDDING
+
+# 2. Check vector store provider
+cat .env | grep VECTOR_STORE_PROVIDER
+```
+
+### Step 2: Test Embedding Generation
+```python
+# Test script to check embedding dimensions
+from services.vector_store import vector_store_service
+
+# Generate a test embedding
+test_embedding = vector_store_service.embeddings.embed_query("test")
+print(f"Current embedding dimensions: {len(test_embedding)}")
+```
+
+### Step 3: Check Database Content
+For MongoDB users:
+```javascript
+// MongoDB shell command to check stored embedding dimensions
+db.your_collection.findOne({"ingredients_emb": {"$exists": true}})
+```
+
+### Step 4: Apply Fix
+Choose one of the strategies above based on your needs.
+
+## 📝 Best Practices
+
+### 1. Document Your Embedding Model
+Keep a record of which embedding model you used:
+```bash
+# Add comments to your .env file
+# Database created on 2025-08-27 with all-MiniLM-L6-v2 (384D)
+EMBEDDING_PROVIDER=huggingface
+HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+```
+
+### 2. Version Control Your Configuration
+```bash
+# Commit your .env changes with descriptive messages
+git add .env
+git commit -m "Update embedding model to match database (384D)"
+```
+
+### 3. Test After Changes
+```bash
+# After changing embedding configuration, test a query
+curl -X POST "http://localhost:8080/chat" \
+  -H "Content-Type: application/json" \
+  -d '{"message": "test query"}'
+```
+
+## 🆘 Quick Reference
+
+### Error Pattern Recognition
+```
+shapes (768,) and (384,) not aligned  → Query=768D, Stored=384D
+shapes (384,) and (768,) not aligned  → Query=384D, Stored=768D
+shapes (1536,) and (384,) not aligned → Query=1536D, Stored=384D
+```
+
+### Quick Fixes
+| Stored Dimensions | Set EMBEDDING_PROVIDER to |
+|-------------------|-------------------------|
+| 384 | `huggingface` with `all-MiniLM-L6-v2` |
+| 768 | `ollama` with `nomic-embed-text:v1.5` |
+| 1536 | `openai` with `text-embedding-3-small` |
+
+---
+
+## 📞 Need Help?
+
+If you're still experiencing issues:
+
+1. Check the application logs for detailed error messages
+2. Verify your embedding model is properly installed/accessible
+3. Ensure your database connection is working
+4. Consider regenerating your vector database if switching models permanently
+
+Remember: **Consistency is key** - your query embeddings and stored embeddings must use the same model and dimensions!

backend/docs/embedding-troubleshooting.md

@@ -0,0 +1,132 @@
+# 🚨 Embedding Troubleshooting Quick Start
+
+## Common Error Messages & Instant Fixes
+
+### ⚠️ "shapes (768,) and (384,) not aligned"
+
+**What it means:** Your query embeddings (768D) don't match stored embeddings (384D)
+
+**Instant fix:**
+```bash
+# Open .env file and change:
+EMBEDDING_PROVIDER=huggingface
+HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+
+# Restart your application
+```
+
+### ⚠️ "shapes (384,) and (768,) not aligned"
+
+**What it means:** Your query embeddings (384D) don't match stored embeddings (768D)
+
+**Instant fix:**
+```bash
+# Open .env file and change:
+EMBEDDING_PROVIDER=ollama
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text:v1.5
+
+# Make sure Ollama is running: ollama serve
+# Pull the model: ollama pull nomic-embed-text:v1.5
+# Restart your application
+```
+
+### ⚠️ "shapes (1536,) and (384,) not aligned"
+
+**What it means:** Your query embeddings (1536D) don't match stored embeddings (384D)
+
+**Instant fix:**
+```bash
+# Open .env file and change:
+EMBEDDING_PROVIDER=huggingface
+HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+
+# Restart your application
+```
+
+## 🔧 5-Minute Fix Guide
+
+### Step 1: Identify Your Error (30 seconds)
+Look at your error message and find the dimension numbers:
+- `shapes (X,) and (Y,)` → X = query dimensions, Y = stored dimensions
+
+### Step 2: Choose Matching Model (1 minute)
+| Stored Dimensions (Y) | Set in .env |
+|---------------------|-------------|
+| 384 | `EMBEDDING_PROVIDER=huggingface`<br/>`HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2` |
+| 768 | `EMBEDDING_PROVIDER=ollama`<br/>`OLLAMA_EMBEDDING_MODEL=nomic-embed-text:v1.5` |
+| 1024 | `EMBEDDING_PROVIDER=ollama`<br/>`OLLAMA_EMBEDDING_MODEL=mxbai-embed-large` |
+| 1536 | `EMBEDDING_PROVIDER=openai`<br/>`OPENAI_EMBEDDING_MODEL=text-embedding-3-small` |
+
+### Step 3: Update Configuration (2 minutes)
+```bash
+# Edit your .env file
+nano .env  # or use your preferred editor
+
+# Find the EMBEDDING_PROVIDER lines and update them
+# Save the file
+```
+
+### Step 4: Restart Application (1 minute)
+```bash
+# Kill current process (Ctrl+C)
+# Restart
+uvicorn app:app --reload
+```
+
+### Step 5: Test (30 seconds)
+```bash
+# Test with a simple query
+curl -X POST "http://localhost:8080/chat" \
+  -H "Content-Type: application/json" \
+  -d '{"message": "chicken recipe"}'
+```
+
+## 🔍 Alternative: Start Fresh
+
+If you prefer to use a different embedding model permanently:
+
+### Option A: Regenerate Database (5 minutes)
+```bash
+# 1. Choose your preferred model in .env
+EMBEDDING_PROVIDER=ollama
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text:v1.5
+
+# 2. Enable database refresh
+DB_REFRESH_ON_START=true
+
+# 3. Restart application (this will rebuild everything)
+uvicorn app:app --reload
+
+# 4. IMPORTANT: Disable refresh after startup
+DB_REFRESH_ON_START=false
+```
+
+### Option B: Switch Vector Store (2 minutes)
+```bash
+# Switch to ChromaDB (will create fresh database)
+VECTOR_STORE_PROVIDER=chromadb
+
+# Restart application
+uvicorn app:app --reload
+```
+
+## ⚡ Prevention Tips
+
+### Document Your Choice
+Add a comment to your .env file:
+```bash
+# Created 2025-08-27 with all-MiniLM-L6-v2 (384 dimensions)
+EMBEDDING_PROVIDER=huggingface
+HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+```
+
+### Consistent Development
+If working in a team, ensure everyone uses the same configuration:
+```bash
+# Share this in your team chat:
+# "Use EMBEDDING_PROVIDER=huggingface with all-MiniLM-L6-v2"
+```
+
+---
+
+**Still stuck?** Check the full [Embedding Compatibility Guide](./embedding-compatibility-guide.md) for detailed explanations.

backend/docs/logging_guide.md

@@ -0,0 +1,56 @@
+# Logging Example and Configuration
+
+## Logging Features Implemented
+
+### 1. Centralized Logging Configuration
+- **File**: `config/logging_config.py`
+- **Features**: 
+  - Rotating file logs (10MB max, 5 backups)
+  - Console and file output
+  - Structured format with timestamps, levels, and source location
+  - Environment-based configuration
+
+### 2. Service-Level Logging
+- **Vector Store Service**: Logs initialization, document loading, provider setup
+- **LLM Service**: Logs model setup, question processing, memory operations
+- **API Endpoints**: Logs requests, responses, and errors
+
+### 3. Log Levels Used
+- **INFO**: Normal operations, successful completions
+- **DEBUG**: Detailed operation steps, memory operations
+- **WARNING**: Non-critical issues, fallbacks
+- **ERROR**: Failures, exceptions (with stack traces)
+
+### 4. Emoji-Coded Log Messages
+- 🚀 Startup/Initialization
+- ✅ Success operations
+- ❌ Error conditions
+- ⚠️ Warning conditions
+- 🔧 Configuration/Setup
+- 💬 Chat operations
+- 🧠 Memory operations
+- 📊 Database operations
+- 🔍 Search/Retrieval
+- 💾 Storage operations
+
+## Usage Examples
+
+```python
+from config.logging_config import get_logger
+
+# Get a logger for your module
+logger = get_logger("my_module")
+
+# Log at different levels
+logger.info("✅ Operation completed successfully")
+logger.warning("⚠️ Using fallback configuration")
+logger.error("❌ Operation failed", exc_info=True)  # Includes stack trace
+```
+
+## Log File Location
+- **Path**: `./logs/recipe_bot.log`
+- **Rotation**: Automatic when file exceeds 10MB
+- **Backups**: Keeps 5 backup files
+
+## Console Output
+All logs are also displayed in the console with colored formatting for easy debugging during development.

backend/docs/model-configuration-guide.md

@@ -0,0 +1,542 @@
+# Model Configuration Guide
+
+This guide focuses on the technical configuration, settings management, parameter handling, and troubleshooting for LLM providers in the Recipe Chatbot project.
+
+> 📚 **Looking for model recommendations?** See [Model Selection Guide](./model-selection-guide.md) for detailed model comparisons and use case recommendations.
+
+## 🔧 Configuration System Overview
+
+### Settings Architecture
+The project uses a centralized configuration system in `config/settings.py` with environment variable overrides:
+
+```python
+# Configuration loading flow
+Environment Variables (.env) → settings.py → LLM Service → Provider APIs
+```
+
+### Temperature Management
+Each provider has different temperature constraints that are automatically handled:
+
+| Provider | Range | Auto-Handling | Special Cases |
+|----------|-------|---------------|---------------|
+| **OpenAI** | 0.0 - 2.0 | ✅ GPT-5-nano → 1.0 | Nano models fixed |
+| **Google** | 0.0 - 1.0 | ✅ Clamp to range | Strict validation |
+| **Ollama** | 0.0 - 2.0 | ⚠️ Model dependent | Local processing |
+| **HuggingFace** | Fixed ~0.7 | ❌ API ignores setting | Read-only |
+
+## 🛠️ Provider Configuration Details
+
+### OpenAI Configuration
+
+#### Environment Variables
+```bash
+# Core settings
+OPENAI_API_KEY=sk-proj-xxxxx
+OPENAI_MODEL=gpt-4o-mini
+OPENAI_TEMPERATURE=0.7
+OPENAI_MAX_TOKENS=1000
+
+# Advanced parameters (optional)
+OPENAI_TOP_P=1.0
+OPENAI_FREQUENCY_PENALTY=0.0
+OPENAI_PRESENCE_PENALTY=0.0
+```
+
+#### Automatic Temperature Override
+```python
+# Implemented in services/llm_service.py
+if "gpt-5-nano" in model_name.lower():
+    temperature = 1.0  # Only supported value
+    logger.info(f"Auto-adjusting temperature to 1.0 for {model_name}")
+```
+
+#### Parameter Validation
+- **Temperature**: `0.0 - 2.0` (except nano models: fixed `1.0`)
+- **Max Tokens**: `1 - 4096` (model-dependent)
+- **Top P**: `0.0 - 1.0`
+
+### Google (Gemini) Configuration
+
+#### Environment Variables
+```bash
+# Core settings
+GOOGLE_API_KEY=AIzaSyxxxxx
+GOOGLE_MODEL=gemini-2.5-flash
+GOOGLE_TEMPERATURE=0.7
+GOOGLE_MAX_TOKENS=1000
+
+# Advanced parameters (optional)
+GOOGLE_TOP_P=0.95
+GOOGLE_TOP_K=40
+```
+
+#### Temperature Clamping
+```python
+# Auto-clamping to Google's range
+google_temp = max(0.0, min(1.0, configured_temperature))
+if google_temp != configured_temperature:
+    logger.info(f"Clamping temperature from {configured_temperature} to {google_temp}")
+```
+
+#### Parameter Constraints
+- **Temperature**: `0.0 - 1.0` (strictly enforced)
+- **Max Tokens**: `1 - 8192`
+- **Top K**: `1 - 40`
+
+### Ollama Configuration
+
+#### Environment Variables
+```bash
+# Core settings
+OLLAMA_BASE_URL=http://localhost:11434
+OLLAMA_MODEL=llama3.1:8b
+OLLAMA_TEMPERATURE=0.7
+OLLAMA_MAX_TOKENS=1000
+
+# Connection settings
+OLLAMA_TIMEOUT=30
+OLLAMA_KEEP_ALIVE=5m
+```
+
+#### Service Management
+```bash
+# Start Ollama service
+ollama serve &
+
+# Verify service status
+curl http://localhost:11434/api/version
+
+# Model management
+ollama pull llama3.1:8b
+ollama list
+ollama rm unused_model
+```
+
+#### Parameter Flexibility
+- **Temperature**: `0.0 - 2.0` (widest range)
+- **Context Length**: Model-dependent (2K - 128K)
+- **Custom Parameters**: Model-specific options available
+
+### HuggingFace Configuration
+
+#### Environment Variables
+```bash
+# Core settings
+HUGGINGFACE_API_KEY=hf_xxxxx
+HUGGINGFACE_MODEL=microsoft/DialoGPT-medium
+HUGGINGFACE_TEMPERATURE=0.7  # Often ignored
+HUGGINGFACE_MAX_TOKENS=500
+
+# API settings
+HUGGINGFACE_WAIT_FOR_MODEL=true
+HUGGINGFACE_USE_CACHE=true
+```
+
+#### API Limitations
+```python
+# Note: Temperature is often ignored by Inference API
+logger.warning(f"HuggingFace model {model_name} may ignore temperature setting")
+return 0.7  # API typically uses this default
+```
+
+## ⚙️ Advanced Configuration
+
+### Dynamic Provider Switching
+```python
+# config/settings.py implementation
+def get_llm_config():
+    provider = os.getenv("LLM_PROVIDER", "openai").lower()
+    fallback = os.getenv("LLM_FALLBACK_PROVIDER", "google").lower()
+    
+    return {
+        "provider": provider,
+        "fallback_provider": fallback,
+        **get_provider_config(provider)
+    }
+
+def get_provider_config(provider):
+    """Get provider-specific configuration."""
+    configs = {
+        "openai": {
+            "api_key": os.getenv("OPENAI_API_KEY"),
+            "model": os.getenv("OPENAI_MODEL", "gpt-4o-mini"),
+            "temperature": float(os.getenv("OPENAI_TEMPERATURE", "0.7")),
+            "max_tokens": int(os.getenv("OPENAI_MAX_TOKENS", "1000")),
+        },
+        "google": {
+            "api_key": os.getenv("GOOGLE_API_KEY"),
+            "model": os.getenv("GOOGLE_MODEL", "gemini-2.5-flash"),
+            "temperature": float(os.getenv("GOOGLE_TEMPERATURE", "0.7")),
+            "max_tokens": int(os.getenv("GOOGLE_MAX_TOKENS", "1000")),
+        },
+        # ... other providers
+    }
+    return configs.get(provider, {})
+```
+
+### Fallback Configuration
+```python
+# Automatic fallback on provider failure
+def get_llm_response(message):
+    try:
+        return primary_provider.chat_completion(message)
+    except Exception as e:
+        logger.warning(f"Primary provider failed: {e}")
+        return fallback_provider.chat_completion(message)
+```
+
+### Environment-Specific Configs
+
+#### Development (.env.development)
+```bash
+# Fast, free/cheap for testing
+LLM_PROVIDER=google
+GOOGLE_MODEL=gemini-2.5-flash
+GOOGLE_TEMPERATURE=0.8  # More creative for testing
+LLM_FALLBACK_PROVIDER=ollama
+```
+
+#### Production (.env.production)
+```bash
+# Reliable, consistent for production
+LLM_PROVIDER=openai
+OPENAI_MODEL=gpt-4o-mini
+OPENAI_TEMPERATURE=0.7  # Consistent responses
+LLM_FALLBACK_PROVIDER=google
+```
+
+#### Local Development (.env.local)
+```bash
+# Self-hosted for offline development
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=llama3.1:8b
+OLLAMA_TEMPERATURE=0.7
+# No fallback - fully local
+```
+
+## 🚨 Configuration Troubleshooting
+
+### Issue: GPT-5-nano Temperature Error
+**Error**: `Temperature must be 1.0 for gpt-5-nano`
+**Status**: ✅ Auto-fixed in `services/llm_service.py`
+**Verification**:
+```bash
+python -c "
+import os
+os.environ['OPENAI_MODEL'] = 'gpt-5-nano'
+os.environ['OPENAI_TEMPERATURE'] = '0.5'
+from services.llm_service import LLMService
+LLMService()  # Should log temperature override
+"
+```
+
+### Issue: Google Temperature Out of Range
+**Error**: `Temperature must be between 0.0 and 1.0`
+**Solution**: Automatic clamping implemented
+**Test**:
+```bash
+python -c "
+import os
+os.environ['LLM_PROVIDER'] = 'google'
+os.environ['GOOGLE_TEMPERATURE'] = '1.5'
+from services.llm_service import LLMService
+LLMService()  # Should clamp to 1.0
+"
+```
+
+### Issue: Ollama Connection Failed
+**Error**: `ConnectionError: Could not connect to Ollama`
+**Diagnosis**:
+```bash
+# Check if Ollama is running
+curl -f http://localhost:11434/api/version || echo "Ollama not running"
+
+# Check if model exists
+ollama list | grep "llama3.1:8b" || echo "Model not found"
+
+# Check system resources
+free -h  # RAM usage
+df -h    # Disk space
+```
+
+**Fix**:
+```bash
+# Start Ollama service
+ollama serve &
+
+# Pull required model
+ollama pull llama3.1:8b
+
+# Test connection
+curl -d '{"model":"llama3.1:8b","prompt":"test","stream":false}' \
+     http://localhost:11434/api/generate
+```
+
+### Issue: HuggingFace Temperature Ignored
+**Issue**: Settings have no effect on response
+**Explanation**: This is expected behavior - HuggingFace Inference API typically ignores temperature
+**Workaround**: Use different models or providers for temperature control
+
+### Issue: Missing API Keys
+**Error**: `AuthenticationError: Invalid API key`
+**Diagnosis**:
+```bash
+# Check environment variables
+echo "OpenAI: ${OPENAI_API_KEY:0:10}..." 
+echo "Google: ${GOOGLE_API_KEY:0:10}..."
+echo "HuggingFace: ${HUGGINGFACE_API_KEY:0:10}..."
+
+# Test API key validity
+curl -H "Authorization: Bearer $OPENAI_API_KEY" \
+     https://api.openai.com/v1/models | jq '.data[0].id' || echo "Invalid OpenAI key"
+```
+
+## 🔍 Configuration Validation
+
+### Automated Configuration Check
+```bash
+# Run comprehensive configuration validation
+python -c "
+from config.settings import get_llm_config
+from services.llm_service import LLMService
+import json
+
+print('🔧 Configuration Validation')
+print('=' * 40)
+
+# Load configuration
+try:
+    config = get_llm_config()
+    print('✅ Configuration loaded successfully')
+    print(f'Provider: {config.get(\"provider\")}')
+    print(f'Model: {config.get(\"model\")}')
+    print(f'Temperature: {config.get(\"temperature\")}')
+except Exception as e:
+    print(f'❌ Configuration error: {e}')
+    exit(1)
+
+# Test service initialization
+try:
+    service = LLMService()
+    print('✅ LLM Service initialized')
+except Exception as e:
+    print(f'❌ Service initialization failed: {e}')
+    exit(1)
+
+# Test simple completion
+try:
+    response = service.simple_chat_completion('Test message')
+    print('✅ Chat completion successful')
+    print(f'Response length: {len(response)} characters')
+except Exception as e:
+    print(f'❌ Chat completion failed: {e}')
+    exit(1)
+
+print('🎉 All configuration checks passed!')
+"
+```
+
+### Provider-Specific Health Checks
+```bash
+# OpenAI health check
+curl -H "Authorization: Bearer $OPENAI_API_KEY" \
+     https://api.openai.com/v1/models | jq '.data | length'
+
+# Google health check  
+curl "https://generativelanguage.googleapis.com/v1beta/models?key=$GOOGLE_API_KEY" | jq '.models | length'
+
+# Ollama health check
+curl http://localhost:11434/api/tags | jq '.models | length'
+
+# HuggingFace health check
+curl -H "Authorization: Bearer $HUGGINGFACE_API_KEY" \
+     https://huggingface.co/api/whoami | jq '.name'
+```
+
+### Configuration Diff Tool
+```bash
+# Compare current config with defaults
+python -c "
+import os
+from config.settings import get_llm_config
+
+defaults = {
+    'openai': {'temperature': 0.7, 'max_tokens': 1000},
+    'google': {'temperature': 0.7, 'max_tokens': 1000},
+    'ollama': {'temperature': 0.7, 'max_tokens': 1000},
+}
+
+current = get_llm_config()
+provider = current.get('provider')
+default = defaults.get(provider, {})
+
+print(f'Configuration for {provider}:')
+for key, default_val in default.items():
+    current_val = current.get(key)
+    status = '✅' if current_val == default_val else '⚠️'
+    print(f'{status} {key}: {current_val} (default: {default_val})')
+"
+```
+
+## 📋 Configuration Templates
+
+### Minimal Setup (Single Provider)
+```bash
+# .env.minimal
+LLM_PROVIDER=google
+GOOGLE_API_KEY=your_api_key
+GOOGLE_MODEL=gemini-2.5-flash
+```
+
+### Robust Setup (Primary + Fallback)
+```bash
+# .env.robust  
+LLM_PROVIDER=openai
+OPENAI_API_KEY=your_primary_key
+OPENAI_MODEL=gpt-4o-mini
+LLM_FALLBACK_PROVIDER=google
+GOOGLE_API_KEY=your_fallback_key
+GOOGLE_MODEL=gemini-2.5-flash
+```
+
+### Local-First Setup
+```bash
+# .env.local-first
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=llama3.1:8b
+LLM_FALLBACK_PROVIDER=google
+GOOGLE_API_KEY=your_cloud_backup_key
+```
+
+### Budget-Conscious Setup
+```bash
+# .env.budget
+LLM_PROVIDER=openai
+OPENAI_MODEL=gpt-5-nano
+OPENAI_TEMPERATURE=1.0  # Fixed for nano
+OPENAI_MAX_TOKENS=500   # Reduce costs
+```
+
+## 🔐 Security Best Practices
+
+### API Key Management
+```bash
+# Use environment variables
+export OPENAI_API_KEY="sk-..."
+
+# Never commit keys to git
+echo "*.env*" >> .gitignore
+echo ".env" >> .gitignore
+
+# Use different keys for different environments
+cp .env.example .env.development
+cp .env.example .env.production
+```
+
+### Rate Limiting Configuration
+```python
+# Add to config/settings.py
+RATE_LIMITS = {
+    "openai": {"rpm": 500, "tpm": 40000},
+    "google": {"rpm": 60, "tpm": 32000},
+    "ollama": {"rpm": None, "tpm": None},  # Local = unlimited
+}
+```
+
+### Error Handling Strategy
+```python
+# Graceful degradation configuration
+FALLBACK_CHAIN = [
+    "primary_provider",
+    "fallback_provider", 
+    "local_provider",
+    "cached_response"
+]
+```
+
+## 🧪 Testing Configuration Changes
+
+### Unit Tests for Configuration
+```bash
+# Test temperature overrides
+python -m pytest tests/test_llm_temperature.py -v
+
+# Test provider fallbacks
+python -m pytest tests/test_llm_fallback.py -v
+
+# Test API key validation
+python -m pytest tests/test_api_keys.py -v
+```
+
+### Integration Tests
+```bash
+# Test each provider individually
+python -c "
+import os
+providers = ['openai', 'google', 'ollama']
+
+for provider in providers:
+    os.environ['LLM_PROVIDER'] = provider
+    try:
+        from services.llm_service import LLMService
+        service = LLMService()
+        response = service.simple_chat_completion('Test')
+        print(f'✅ {provider}: {len(response)} chars')
+    except Exception as e:
+        print(f'❌ {provider}: {e}')
+"
+```
+
+### Performance Benchmarks
+```bash
+# Measure response times
+python -c "
+import time
+from services.llm_service import LLMService
+
+service = LLMService()
+start = time.time()
+response = service.simple_chat_completion('Quick recipe suggestion')
+elapsed = time.time() - start
+
+print(f'Response time: {elapsed:.2f}s')
+print(f'Response length: {len(response)} characters')
+print(f'Words per second: {len(response.split()) / elapsed:.1f}')
+"
+```
+
+## 🔄 Configuration Migration
+
+### Upgrading from Old Configuration
+```bash
+# Migrate old environment variables
+# Old format → New format
+mv .env .env.backup
+
+# Update variable names
+sed 's/LLM_MODEL=/OPENAI_MODEL=/' .env.backup > .env
+sed -i 's/LLM_TEMPERATURE=/OPENAI_TEMPERATURE=/' .env
+sed -i 's/LLM_MAX_TOKENS=/OPENAI_MAX_TOKENS=/' .env
+
+echo "LLM_PROVIDER=openai" >> .env
+```
+
+### Version Compatibility Check
+```python
+# Check if configuration is compatible
+def check_config_version():
+    required_vars = ["LLM_PROVIDER"]
+    legacy_vars = ["LLM_MODEL", "LLM_TEMPERATURE"]
+    
+    has_new = all(os.getenv(var) for var in required_vars)
+    has_legacy = any(os.getenv(var) for var in legacy_vars)
+    
+    if has_legacy and not has_new:
+        raise ValueError("Legacy configuration detected. Please migrate to new format.")
+    
+    return has_new
+```
+
+---
+
+💡 **Next Steps**: After configuring your providers, see the [Model Selection Guide](./model-selection-guide.md) for choosing the best models for your use case.

backend/docs/model-selection-guide.md

@@ -0,0 +1,502 @@
+# Model Selection Guide
+
+## 🎯 At-a-Glance Recommendations
+
+| Priority | Best Choice | Provider | Monthly Cost* | Setup Time | Quality Score | Why Choose This |
+|----------|-------------|----------|---------------|------------|---------------|-----------------|
+| **Ease of Use** | Gemini 2.5 Flash | Google | Free - $2 | 2 min | 90% | Excellent free tier |
+| **Best Value** | GPT-5-nano | OpenAI | $1.00 | 2 min | 88% | Modern GPT-5 at nano price |
+| **Premium Quality** | Claude 3 Opus | Anthropic | $225 | 2 min | 95% | Highest reasoning quality |
+| **Self-Hosted** | Llama 3.1:8b | Ollama | Free | 10 min | 82% | Perfect balance |
+| **High-End Local** | DeepSeek-R1:7b | Ollama | Free | 15 min | 88% | Best reasoning model |
+| **Budget Cloud** | Claude 3.5 Haiku | Anthropic | $4 | 2 min | 87% | Fast and affordable |
+| **Alternative Local** | CodeQwen1.5:7b | Ollama | Free | 10 min | 85% | Excellent for structured data |
+
+*Based on 30,000 queries/month
+
+---
+
+## 🏢 Cloud Models (Closed Source)
+
+### OpenAI Models
+
+#### GPT-5 (Latest Flagship) ⭐ **NEW**
+```bash
+OPENAI_MODEL=gpt-5
+```
+- **Pricing**: $20/month (Plus plan) - Unlimited with guardrails
+- **Capabilities**: Advanced reasoning, thinking, code execution
+- **Best For**: Premium applications requiring cutting-edge AI
+- **Recipe Quality**: Outstanding (96%) - Best culinary understanding
+- **Context**: 196K tokens (reasoning mode)
+
+
+#### GPT-5-nano (Ultra Budget) ⭐ **MISSED GEM**
+```bash
+OPENAI_MODEL=gpt-5-nano
+```
+- **Pricing**: $0.05/1M input, $0.40/1M output tokens
+- **Monthly Cost**: ~$1.00 for 30K queries
+- **Best For**: Budget-conscious deployments with modern capabilities
+- **Recipe Quality**: Very Good (88%)
+- **Speed**: Very Fast
+- **Features**: GPT-5 architecture at nano pricing
+
+
+#### GPT-4o-mini (Proven Budget Choice)
+```bash
+OPENAI_MODEL=gpt-4o-mini
+```
+- **Pricing**: $0.15/1M input, $0.60/1M output tokens
+- **Monthly Cost**: ~$4 for 30K queries
+- **Best For**: Cost-effective production deployments
+- **Recipe Quality**: Very Good (86%)
+- **Speed**: Very Fast
+
+
+### Google AI (Gemini) Models
+
+#### Gemini 2.5 Flash ⭐ **RECOMMENDED**
+```bash
+GOOGLE_MODEL=gemini-2.5-flash
+```
+- **Pricing**: Free tier, then $0.30/1M input, $2.50/1M output
+- **Monthly Cost**: Free - $2 for most usage patterns
+- **Best For**: Development and cost-conscious production
+- **Recipe Quality**: Excellent (90%)
+- **Features**: Thinking budgets, 1M context window
+
+#### Gemini 2.5 Pro (High-End)
+```bash
+GOOGLE_MODEL=gemini-2.5-pro
+```
+- **Pricing**: $1.25/1M input, $10/1M output (≤200K context)
+- **Monthly Cost**: ~$25 for 30K queries
+- **Best For**: Premium applications requiring best Google AI
+- **Recipe Quality**: Excellent (92%)
+
+#### Gemini 2.0 Flash-Lite (Ultra Budget)
+```bash
+GOOGLE_MODEL=gemini-2.0-flash-lite
+```
+- **Pricing**: $0.075/1M input, $0.30/1M output
+- **Monthly Cost**: ~$0.90 for 30K queries
+- **Best For**: High-volume, cost-sensitive applications
+- **Recipe Quality**: Good (85%)
+
+
+## 🔓 Open Source Models (Self-Hosted)
+
+### Ollama Models (Latest Releases)
+
+#### DeepSeek-R1:7b ⭐ **BREAKTHROUGH MODEL**
+```bash
+OLLAMA_MODEL=deepseek-r1:7b
+```
+- **Parameters**: 7B
+- **Download**: ~4.7GB
+- **RAM Required**: 8GB
+- **Best For**: Advanced reasoning tasks, O1-level performance
+- **Recipe Quality**: Outstanding (88%)
+- **Special**: Chain-of-thought reasoning, approaching GPT-4 performance
+
+#### Gemma 3:27b ⭐ **NEW FLAGSHIP**
+```bash
+OLLAMA_MODEL=gemma3:27b
+```
+- **Parameters**: 27B
+- **Download**: ~17GB
+- **RAM Required**: 32GB
+- **Best For**: Highest quality open source experience
+- **Recipe Quality**: Outstanding (89%)
+- **Features**: Vision capabilities, state-of-the-art performance
+
+#### Llama 3.1:8b (Proven Choice)
+```bash
+OLLAMA_MODEL=llama3.1:8b
+```
+- **Parameters**: 8B
+- **Download**: ~4.7GB
+- **RAM Required**: 8GB
+- **Best For**: Balanced production deployment
+- **Recipe Quality**: Very Good (82%)
+- **Status**: Your current choice - excellent balance!
+
+#### Qwen 3:8b ⭐ **NEW RELEASE**
+```bash
+OLLAMA_MODEL=qwen3:8b
+```
+- **Parameters**: 8B
+- **Download**: ~4.4GB
+- **RAM Required**: 8GB
+- **Best For**: Multilingual support, latest technology
+- **Recipe Quality**: Very Good (84%)
+- **Features**: Tool use, thinking capabilities
+
+#### Phi 4:14b ⭐ **MICROSOFT'S LATEST**
+```bash
+OLLAMA_MODEL=phi4:14b
+```
+- **Parameters**: 14B
+- **Download**: ~9.1GB
+- **RAM Required**: 16GB
+- **Best For**: Reasoning and math tasks
+- **Recipe Quality**: Very Good (85%)
+- **Features**: State-of-the-art efficiency
+
+#### Gemma 3:4b (Efficient Choice)
+```bash
+OLLAMA_MODEL=gemma3:4b
+```
+- **Parameters**: 4B
+- **Download**: ~3.3GB
+- **RAM Required**: 6GB
+- **Best For**: Resource-constrained deployments
+- **Recipe Quality**: Good (78%)
+- **Features**: Excellent for size, runs on modest hardware
+
+### HuggingFace Models (Downloadable for Local Use)
+
+#### CodeQwen1.5:7b ⭐ **ALIBABA'S CODE MODEL**
+```bash
+OLLAMA_MODEL=codeqwen:7b
+```
+- **Parameters**: 7B
+- **Download**: ~4.2GB
+- **RAM Required**: 8GB
+- **Best For**: Recipe parsing, ingredient analysis, structured data
+- **Recipe Quality**: Very Good (85%)
+- **Features**: Excellent at understanding structured recipe formats
+
+#### Mistral-Nemo:12b ⭐ **BALANCED CHOICE**
+```bash
+OLLAMA_MODEL=mistral-nemo:12b
+```
+- **Parameters**: 12B
+- **Download**: ~7GB
+- **RAM Required**: 12GB
+- **Best For**: General conversation with good reasoning
+- **Recipe Quality**: Very Good (84%)
+- **Features**: Multilingual, efficient, well-balanced
+
+#### Nous-Hermes2:10.7b ⭐ **FINE-TUNED EXCELLENCE**
+```bash
+OLLAMA_MODEL=nous-hermes2:10.7b
+```
+- **Parameters**: 10.7B
+- **Download**: ~6.4GB
+- **RAM Required**: 12GB
+- **Best For**: Instruction following, detailed responses
+- **Recipe Quality**: Very Good (83%)
+- **Features**: Excellent instruction following, helpful responses
+
+#### OpenHermes2.5-Mistral:7b ⭐ **COMMUNITY FAVORITE**
+```bash
+OLLAMA_MODEL=openhermes2.5-mistral:7b
+```
+- **Parameters**: 7B
+- **Download**: ~4.1GB
+- **RAM Required**: 8GB
+- **Best For**: Creative recipe suggestions, conversational AI
+- **Recipe Quality**: Good (81%)
+- **Features**: Creative, conversational, reliable
+
+#### Solar:10.7b ⭐ **UPSTAGE'S MODEL**
+```bash
+OLLAMA_MODEL=solar:10.7b
+```
+- **Parameters**: 10.7B
+- **Download**: ~6.1GB
+- **RAM Required**: 12GB
+- **Best For**: Analytical tasks, recipe modifications
+- **Recipe Quality**: Very Good (83%)
+- **Features**: Strong analytical capabilities, detailed explanations
+
+
+### Anthropic Claude Models
+
+#### Claude 3.5 Sonnet (Production Standard)
+```bash
+ANTHROPIC_MODEL=claude-3-5-sonnet-20241022
+```
+- **Pricing**: $3/1M input, $15/1M output tokens
+- **Monthly Cost**: ~$45 for 30K queries
+- **Best For**: Balanced performance and reasoning
+- **Recipe Quality**: Outstanding (94%)
+- **Features**: Advanced analysis, code understanding
+
+#### Claude 3.5 Haiku (Speed Focused)
+```bash
+ANTHROPIC_MODEL=claude-3-5-haiku-20241022
+```
+- **Pricing**: $0.25/1M input, $1.25/1M output tokens
+- **Monthly Cost**: ~$4 for 30K queries
+- **Best For**: Fast, cost-effective responses
+- **Recipe Quality**: Very Good (87%)
+- **Features**: Lightning fast, good quality
+
+#### Claude 3 Opus (Premium Reasoning)
+```bash
+ANTHROPIC_MODEL=claude-3-opus-20240229
+```
+- **Pricing**: $15/1M input, $75/1M output tokens
+- **Monthly Cost**: ~$225 for 30K queries
+- **Best For**: Complex reasoning, highest quality
+- **Recipe Quality**: Outstanding (95%)
+- **Features**: Top-tier reasoning, complex tasks
+
+---
+
+
+## 🎯 Scenario-Based Recommendations
+
+### 👨‍💻 **Development & Testing**
+**Choice**: Gemini 2.5 Flash
+```bash
+LLM_PROVIDER=google
+GOOGLE_MODEL=gemini-2.5-flash
+```
+- Free tier covers most development
+- Excellent quality for testing
+- Easy setup and integration
+
+### 🚀 **Small to Medium Production**
+**Choice**: Gemini 2.5 Flash or GPT-4o-mini
+```bash
+# Cost-focused
+LLM_PROVIDER=google
+GOOGLE_MODEL=gemini-2.5-flash
+
+# Quality-focused
+LLM_PROVIDER=openai
+OPENAI_MODEL=gpt-4o-mini
+```
+
+### 🏠 **Self-Hosted**
+**Choice**: Llama 3.1:8b or upgrade to DeepSeek-R1:7b
+```bash
+# Your current (excellent choice)
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=llama3.1:8b
+
+# Upgrade option (better reasoning)
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=deepseek-r1:7b
+```
+
+### 💰 **Budget/Free**
+**Choice**: Local models or GPT-5-nano
+```bash
+# Best local alternative
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=codeqwen:7b
+
+# Best budget paid option
+LLM_PROVIDER=openai
+OPENAI_MODEL=gpt-5-nano
+
+# Quality budget cloud
+LLM_PROVIDER=anthropic
+ANTHROPIC_MODEL=claude-3-5-haiku-20241022
+```
+
+### 🔒 **Privacy/Offline**
+**Choice**: DeepSeek-R1:7b or Gemma 3:4b
+```bash
+# Best reasoning
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=deepseek-r1:7b
+
+# Resource-efficient
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=gemma3:4b
+```
+
+---
+
+## ⚡ Quick Setup Commands
+
+### Cloud Models (Instant Setup)
+
+#### Gemini 2.5 Flash (Recommended)
+```bash
+# Update .env
+LLM_PROVIDER=google
+GOOGLE_MODEL=gemini-2.5-flash
+GOOGLE_TEMPERATURE=0.7
+GOOGLE_MAX_TOKENS=1000
+
+# Test
+python -c "
+from services.llm_service import LLMService
+service = LLMService()
+print('✅ Gemini 2.5 Flash ready!')
+response = service.simple_chat_completion('Suggest a quick pasta recipe')
+print(f'Response: {response[:100]}...')
+"
+```
+
+#### CodeQwen1.5:7b (Structured Data Expert)
+```bash
+# Pull model
+ollama pull codeqwen:7b
+
+# Update .env
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=codeqwen:7b
+OLLAMA_TEMPERATURE=0.7
+
+# Test
+python -c "
+from services.llm_service import LLMService
+service = LLMService()
+print('✅ CodeQwen 1.5:7b ready!')
+response = service.simple_chat_completion('Parse this recipe: 2 cups flour, 1 egg, 1 cup milk')
+print(f'Response: {response[:100]}...')
+"
+```
+
+#### Mistral-Nemo:12b (Balanced Performance)
+```bash
+# Pull model
+ollama pull mistral-nemo:12b
+
+# Update .env
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=mistral-nemo:12b
+OLLAMA_TEMPERATURE=0.7
+
+# Test
+python -c "
+from services.llm_service import LLMService
+service = LLMService()
+print('✅ Mistral-Nemo ready!')
+response = service.simple_chat_completion('Suggest a Mediterranean dinner menu')
+print(f'Response: {response[:100]}...')
+"
+```
+
+#### Claude 3.5 Haiku (Speed + Quality)
+```bash
+# Update .env
+LLM_PROVIDER=anthropic
+ANTHROPIC_MODEL=claude-3-5-haiku-20241022
+ANTHROPIC_TEMPERATURE=0.7
+ANTHROPIC_MAX_TOKENS=1000
+
+# Test
+python -c "
+from services.llm_service import LLMService
+service = LLMService()
+print('✅ Claude 3.5 Haiku ready!')
+response = service.simple_chat_completion('Quick dinner ideas with vegetables')
+print(f'Response: {response[:100]}...')
+"
+```
+
+#### GPT-5-nano (Budget Winner)
+```bash
+# Update .env
+LLM_PROVIDER=openai
+OPENAI_MODEL=gpt-5-nano
+OPENAI_TEMPERATURE=0.7
+OPENAI_MAX_TOKENS=1000
+
+# Test
+python -c "
+from services.llm_service import LLMService
+service = LLMService()
+print('✅ GPT-5-nano ready!')
+response = service.simple_chat_completion('Quick healthy breakfast ideas')
+print(f'Response: {response[:100]}...')
+"
+```
+
+#### GPT-5 (Premium)
+```bash
+# Update .env
+LLM_PROVIDER=openai
+OPENAI_MODEL=gpt-5
+OPENAI_TEMPERATURE=0.7
+OPENAI_MAX_TOKENS=1000
+
+# Test
+python -c "
+from services.llm_service import LLMService
+service = LLMService()
+print('✅ GPT-5 ready!')
+response = service.simple_chat_completion('Create a healthy meal plan')
+print(f'Response: {response[:100]}...')
+"
+```
+
+### Self-Hosted Models
+
+#### DeepSeek-R1:7b (Latest Breakthrough)
+```bash
+# Pull model
+ollama pull deepseek-r1:7b
+
+# Update .env
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=deepseek-r1:7b
+OLLAMA_TEMPERATURE=0.7
+
+# Start Ollama
+ollama serve &
+
+# Test
+python -c "
+from services.llm_service import LLMService
+service = LLMService()
+print('✅ DeepSeek-R1 ready!')
+response = service.simple_chat_completion('Explain the science behind sourdough fermentation')
+print(f'Response: {response[:100]}...')
+"
+```
+
+#### Gemma 3:4b (Efficient)
+```bash
+# Pull model
+ollama pull gemma3:4b
+
+# Update .env
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=gemma3:4b
+OLLAMA_TEMPERATURE=0.7
+
+# Test
+python -c "
+from services.llm_service import LLMService
+service = LLMService()
+print('✅ Gemma 3:4b ready!')
+response = service.simple_chat_completion('Quick chicken recipes for weeknight dinners')
+print(f'Response: {response[:100]}...')
+"
+```
+
+---
+
+## 🔧 Hardware Requirements
+
+### Cloud Models
+- **Requirements**: Internet connection, API key
+- **RAM**: Any (processing done remotely)
+- **Storage**: Minimal
+- **Best For**: Instant setup, no hardware constraints
+
+### Self-Hosted Requirements
+
+| Model | Parameters | RAM Needed | Storage | GPU Beneficial | Best For |
+|-------|------------|------------|---------|----------------|----------|
+| `gemma3:4b` | 4B | 6GB | 3.3GB | Optional | Laptops, modest hardware |
+| `codeqwen:7b` | 7B | 8GB | 4.2GB | Yes | Structured data, parsing |
+| `llama3.1:8b` | 8B | 8GB | 4.7GB | Yes | Standard workstations |
+| `deepseek-r1:7b` | 7B | 8GB | 4.7GB | Yes | Reasoning tasks |
+| `openhermes2.5-mistral:7b` | 7B | 8GB | 4.1GB | Yes | Conversational AI |
+| `nous-hermes2:10.7b` | 10.7B | 12GB | 6.4GB | Recommended | Instruction following |
+| `mistral-nemo:12b` | 12B | 12GB | 7GB | Recommended | Balanced performance |
+| `phi4:14b` | 14B | 16GB | 9.1GB | Recommended | High-end workstations |
+| `gemma3:27b` | 27B | 32GB | 17GB | Required | Powerful servers |
+
+---

backend/docs/opensource-llm-configuration.md

@@ -0,0 +1,394 @@
+# Open Source LLM Configuration Guide (HuggingFace & Ollama)
+
+## Overview
+The Recipe Recommendation Bot supports open source models through both HuggingFace and Ollama. This guide explains how to configure these providers for optimal performance, with recommended models under 20B parameters.
+
+> 📚 **For comprehensive model comparisons including closed source options (OpenAI, Google), see [Comprehensive Model Guide](./comprehensive-model-guide.md)**
+
+## Quick Model Recommendations
+
+| Use Case | Model | Download Size | RAM Required | Quality |
+|----------|-------|---------------|--------------|---------|
+| **Development** | `gemma2:2b` | 1.6GB | 4GB | Good |
+| **Production** | `llama3.1:8b` | 4.7GB | 8GB | Excellent |
+| **High Quality** | `llama3.1:13b` | 7.4GB | 16GB | Outstanding |
+| **API (Free)** | `deepseek-ai/DeepSeek-V3.1` | 0GB | N/A | Very Good |
+
+## 🤗 HuggingFace Configuration
+
+### Environment Variables
+
+Add these variables to your `.env` file:
+
+```bash
+# LLM Provider Configuration
+LLM_PROVIDER=huggingface
+
+# HuggingFace Configuration
+HUGGINGFACE_API_TOKEN=your_hf_token_here        # Optional for public models
+HUGGINGFACE_MODEL=deepseek-ai/DeepSeek-V3.1    # Current recommended model
+HUGGINGFACE_API_URL=https://api-inference.huggingface.co/models/
+HUGGINGFACE_USE_API=true                        # Use API vs local inference
+HUGGINGFACE_USE_GPU=false                       # Set to true for local GPU inference
+
+# Embedding Configuration
+HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+```
+
+### Deployment Options
+
+#### Option 1: API Inference (Recommended)
+```bash
+HUGGINGFACE_USE_API=true
+```
+- **Pros**: No local downloads, fast startup, always latest models
+- **Cons**: Requires internet connection, API rate limits
+- **Download Size**: 0 bytes (no local storage needed)
+- **Best for**: Development, testing, quick prototyping
+
+#### Option 2: Local Inference
+```bash
+HUGGINGFACE_USE_API=false
+HUGGINGFACE_USE_GPU=false  # CPU-only
+```
+- **Pros**: No internet required, no rate limits, private
+- **Cons**: Large model downloads, slower inference on CPU
+- **Best for**: Production, offline deployments
+
+#### Option 3: Local GPU Inference
+```bash
+HUGGINGFACE_USE_API=false
+HUGGINGFACE_USE_GPU=true   # Requires CUDA GPU
+```
+- **Pros**: Fast inference, no internet required, no rate limits
+- **Cons**: Large downloads, requires GPU with sufficient VRAM
+- **Best for**: Production with GPU resources
+
+### Recommended HuggingFace Models
+
+#### Lightweight Models (Good for CPU)
+```bash
+HUGGINGFACE_MODEL=microsoft/DialoGPT-small       # ~117MB download
+HUGGINGFACE_MODEL=distilgpt2                     # ~319MB download
+HUGGINGFACE_MODEL=google/flan-t5-small           # ~242MB download
+```
+
+#### Balanced Performance Models
+```bash
+HUGGINGFACE_MODEL=microsoft/DialoGPT-medium      # ~863MB download
+HUGGINGFACE_MODEL=google/flan-t5-base            # ~990MB download
+HUGGINGFACE_MODEL=microsoft/CodeGPT-small-py     # ~510MB download
+```
+
+#### High Quality Models (GPU Recommended)
+```bash
+HUGGINGFACE_MODEL=deepseek-ai/DeepSeek-V3.1      # ~4.2GB download (7B params)
+HUGGINGFACE_MODEL=microsoft/DialoGPT-large       # ~3.2GB download
+HUGGINGFACE_MODEL=google/flan-t5-large           # ~2.8GB download (770M params)
+HUGGINGFACE_MODEL=huggingface/CodeBERTa-small-v1 # ~1.1GB download
+```
+
+#### Specialized Recipe/Cooking Models
+```bash
+HUGGINGFACE_MODEL=recipe-nlg/recipe-nlg-base     # ~450MB download
+HUGGINGFACE_MODEL=cooking-assistant/chef-gpt     # ~2.1GB download (if available)
+```
+
+## 🦙 Ollama Configuration
+
+### Installation
+
+First, install Ollama on your system:
+
+```bash
+# Linux/macOS
+curl -fsSL https://ollama.ai/install.sh | sh
+
+# Windows
+# Download installer from https://ollama.ai/download
+```
+
+### Environment Variables
+
+```bash
+# LLM Provider Configuration
+LLM_PROVIDER=ollama
+
+# Ollama Configuration
+OLLAMA_BASE_URL=http://localhost:11434
+OLLAMA_MODEL=llama3.1:8b
+OLLAMA_TEMPERATURE=0.7
+
+# Embedding Configuration
+OLLAMA_EMBEDDING_MODEL=nomic-embed-text
+```
+
+### Starting Ollama Service
+
+```bash
+# Start Ollama server
+ollama serve
+
+# In another terminal, pull your desired model
+ollama pull llama3.1:8b
+```
+
+### Recommended Ollama Models
+
+#### Lightweight Models (4GB RAM or less)
+```bash
+OLLAMA_MODEL=phi3:mini                  # ~2.3GB download (3.8B params)
+OLLAMA_MODEL=gemma2:2b                  # ~1.6GB download (2B params)
+OLLAMA_MODEL=qwen2:1.5b                 # ~934MB download (1.5B params)
+```
+
+#### Balanced Performance Models (8GB RAM)
+```bash
+OLLAMA_MODEL=llama3.1:8b               # ~4.7GB download (8B params)
+OLLAMA_MODEL=gemma2:9b                 # ~5.4GB download (9B params)
+OLLAMA_MODEL=mistral:7b                # ~4.1GB download (7B params)
+OLLAMA_MODEL=qwen2:7b                  # ~4.4GB download (7B params)
+```
+
+#### High Quality Models (16GB+ RAM)
+```bash
+OLLAMA_MODEL=llama3.1:13b              # ~7.4GB download (13B params)
+OLLAMA_MODEL=mixtral:8x7b              # ~26GB download (47B params - sparse)
+OLLAMA_MODEL=qwen2:14b                 # ~8.2GB download (14B params)
+```
+
+#### Code/Instruction Following Models
+```bash
+OLLAMA_MODEL=codellama:7b              # ~3.8GB download (7B params)
+OLLAMA_MODEL=deepseek-coder:6.7b       # ~3.8GB download (6.7B params)
+OLLAMA_MODEL=wizard-coder:7b           # ~4.1GB download (7B params)
+```
+
+### Ollama Model Management
+
+```bash
+# List available models
+ollama list
+
+# Pull a specific model
+ollama pull llama3.1:8b
+
+# Remove a model to free space
+ollama rm old-model:tag
+
+# Check model information
+ollama show llama3.1:8b
+```
+
+## Installation Requirements
+
+### HuggingFace Setup
+
+#### For API Usage (No Downloads)
+```bash
+pip install -r requirements.txt
+# No additional setup needed
+```
+
+#### For Local CPU Inference
+```bash
+pip install -r requirements.txt
+# Models will be downloaded automatically on first use
+```
+
+#### For Local GPU Inference
+```bash
+# Install CUDA version of PyTorch
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
+
+# Install other requirements
+pip install -r requirements.txt
+
+# Verify GPU availability
+python -c "import torch; print(f'CUDA available: {torch.cuda.is_available()}')"
+```
+
+### Ollama Setup
+
+#### Installation
+```bash
+# Install Ollama
+curl -fsSL https://ollama.ai/install.sh | sh
+
+# Start Ollama service
+ollama serve
+
+# Pull your first model (in another terminal)
+ollama pull llama3.1:8b
+```
+
+## Storage Requirements & Download Sizes
+
+### HuggingFace Local Models
+- **Storage Location**: `~/.cache/huggingface/transformers/`
+- **Small Models**: 100MB - 1GB (good for development)
+- **Medium Models**: 1GB - 5GB (balanced performance)
+- **Large Models**: 5GB - 15GB (high quality, under 20B params)
+
+### Ollama Models
+- **Storage Location**: `~/.ollama/models/`
+- **Quantized Storage**: Models use efficient quantization (4-bit, 8-bit)
+- **2B Models**: ~1-2GB download
+- **7-8B Models**: ~4-5GB download  
+- **13-14B Models**: ~7-8GB download
+
+### Embedding Models
+```bash
+# HuggingFace Embeddings (auto-downloaded)
+sentence-transformers/all-MiniLM-L6-v2     # ~80MB
+sentence-transformers/all-mpnet-base-v2    # ~420MB
+
+# Ollama Embeddings
+ollama pull nomic-embed-text               # ~274MB
+ollama pull mxbai-embed-large              # ~669MB
+```
+
+## Performance & Hardware Recommendations
+
+### System Requirements
+
+#### Minimum (API Usage)
+- **RAM**: 2GB
+- **Storage**: 100MB
+- **Internet**: Required for API calls
+
+#### CPU Inference
+- **RAM**: 8GB+ (16GB for larger models)
+- **CPU**: 4+ cores recommended
+- **Storage**: 5GB+ for models cache
+
+#### GPU Inference
+- **GPU**: 8GB+ VRAM (for 7B models)
+- **RAM**: 16GB+ system RAM
+- **Storage**: 10GB+ for models
+
+### Performance Tips
+
+1. **Start Small**: Begin with lightweight models and upgrade based on quality needs
+2. **Use API First**: Test with HuggingFace API before committing to local inference
+3. **Monitor Resources**: Check CPU/GPU/RAM usage during inference
+4. **Model Caching**: First run downloads models, subsequent runs are faster
+
+## Troubleshooting
+
+### HuggingFace Issues
+
+#### "accelerate package required"
+```bash
+pip install accelerate
+```
+
+#### GPU not detected
+```bash
+# Check CUDA availability
+python -c "import torch; print(torch.cuda.is_available())"
+
+# If false, install CUDA PyTorch
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
+```
+
+#### Out of memory errors
+- Switch to a smaller model
+- Set `HUGGINGFACE_USE_GPU=false` for CPU inference
+- Use API instead: `HUGGINGFACE_USE_API=true`
+
+### Ollama Issues
+
+#### Ollama service not starting
+```bash
+# Check if port 11434 is available
+lsof -i :11434
+
+# Restart Ollama
+ollama serve
+```
+
+#### Model not found
+```bash
+# List available models
+ollama list
+
+# Pull the model
+ollama pull llama3.1:8b
+```
+
+#### Slow inference
+- Try a smaller model
+- Check available RAM
+- Consider using GPU if available
+
+## Quick Tests
+
+### Test HuggingFace Configuration
+```bash
+cd backend
+python -c "
+from services.llm_service import LLMService
+import os
+os.environ['LLM_PROVIDER'] = 'huggingface'
+service = LLMService()
+print('✅ HuggingFace LLM working!')
+response = service.simple_chat_completion('Hello')
+print(f'Response: {response}')
+"
+```
+
+### Test Ollama Configuration
+```bash
+# First ensure Ollama is running
+ollama serve &
+
+# Test the service
+cd backend
+python -c "
+from services.llm_service import LLMService
+import os
+os.environ['LLM_PROVIDER'] = 'ollama'
+service = LLMService()
+print('✅ Ollama LLM working!')
+response = service.simple_chat_completion('Hello')
+print(f'Response: {response}')
+"
+```
+
+## Configuration Examples
+
+### Development Setup (Fast Start)
+```bash
+# Use HuggingFace API for quick testing
+LLM_PROVIDER=huggingface
+HUGGINGFACE_USE_API=true
+HUGGINGFACE_MODEL=deepseek-ai/DeepSeek-V3.1
+HUGGINGFACE_API_TOKEN=your_token_here
+```
+
+### Local CPU Setup
+```bash
+# Local inference on CPU
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=llama3.1:8b
+OLLAMA_BASE_URL=http://localhost:11434
+```
+
+### Local GPU Setup
+```bash
+# Local inference with GPU acceleration
+LLM_PROVIDER=huggingface
+HUGGINGFACE_USE_API=false
+HUGGINGFACE_USE_GPU=true
+HUGGINGFACE_MODEL=deepseek-ai/DeepSeek-V3.1
+```
+
+### Production Setup (High Performance)
+```bash
+# Ollama with optimized model
+LLM_PROVIDER=ollama
+OLLAMA_MODEL=llama3.1:13b  # Higher quality
+OLLAMA_BASE_URL=http://localhost:11434
+# Ensure 16GB+ RAM available
+```

backend/docs/optimal_recipes_structure.md

@@ -0,0 +1,160 @@
+# Universal Recipe Data Structure
+
+This document defines a simple, universal data structure for recipe storage that works efficiently with both ChromaDB and MongoDB Atlas for ingredient-based recipe recommendations.
+
+## Core Principles
+
+1. **Ingredient-focused**: Primary search is by ingredients
+2. **Universal compatibility**: Same structure works for ChromaDB and MongoDB
+3. **Simple and clean**: Easy to understand and maintain
+4. **Efficient retrieval**: Optimized for RAG performance
+
+## Universal Recipe Structure
+
+### Required Fields
+
+```json
+{
+  "title": "String - Recipe name",
+  "ingredients": ["Array of strings - Individual ingredients"], 
+  "instructions": "String - Step-by-step cooking instructions",
+  "metadata": {
+    "cook_time": "String - Optional cooking time",
+    "difficulty": "String - Optional difficulty level",
+    "servings": "String - Optional number of servings",
+    "category": "String - Optional recipe category",
+    "image_url": "String - Optional recipe image URL"
+  }
+}
+```
+
+### Example Document
+
+```json
+{
+  "title": "Mixed Seafood Coconut Fried Rice",
+  "ingredients": [
+    "jasmine rice",
+    "cooked shrimp", 
+    "prawns",
+    "scallops",
+    "coconut milk",
+    "fish sauce",
+    "soy sauce",
+    "garlic",
+    "onion",
+    "ginger",
+    "green onions",
+    "cilantro",
+    "lime",
+    "vegetable oil",
+    "salt",
+    "pepper"
+  ],
+  "instructions": "1. Heat vegetable oil in large pan. 2. Add garlic, onion, ginger and stir-fry until fragrant. 3. Add cooked rice and mix well. 4. Add seafood and cook until heated through. 5. Pour in coconut milk and season with fish sauce and soy sauce. 6. Garnish with green onions and cilantro. 7. Serve with lime wedges.",
+  "metadata": {
+    "cook_time": "25 minutes",
+    "difficulty": "medium", 
+    "servings": "4",
+    "category": "seafood",
+    "image_url": "https://example.com/images/mixed-seafood-coconut-fried-rice.jpg"
+  }
+}
+```
+
+## Key Features
+
+### 1. Clean Ingredients Format
+- **Array structure**: Each ingredient as separate string
+- **Individual embedding**: Each ingredient can be embedded separately
+- **Easy matching**: Simple array operations for ingredient search
+- **No duplicates**: Each ingredient appears once in the array
+
+### 2. Universal Compatibility
+- **ChromaDB**: Automatically creates embeddings from full document
+- **MongoDB Atlas**: Can use pre-computed embeddings or text search
+- **Same structure**: No provider-specific modifications needed
+
+### 3. Efficient Search Patterns
+
+#### Primary: Ingredient-based Search
+```
+User: "I have shrimp, rice, and coconut milk"
+Search: ingredients array for ["shrimp", "rice", "coconut"]
+Result: Mixed Seafood Coconut Fried Rice (high relevance)
+```
+
+#### Secondary: Title-based Search  
+```
+User: "How to make fried rice"
+Search: title field for "fried rice"
+Result: All fried rice recipes
+```
+
+#### Fallback: Full-text Search
+```
+User: "Quick dinner recipes"
+Search: Full document for "quick dinner"
+Result: Recipes mentioning quick preparation
+```
+
+## Implementation Guidelines
+
+### For ChromaDB
+```python
+# Documents are automatically embedded as full text
+ingredients_text = ", ".join(recipe['ingredients'])
+document = Document(
+    page_content=f"Title: {recipe['title']}. Ingredients: {ingredients_text}. Instructions: {recipe['instructions']}",
+    metadata=recipe['metadata']
+)
+```
+
+### For MongoDB Atlas
+```python
+# Can use array search or vector search on the same structure
+# Array search on ingredients
+{"ingredients": {"$in": user_ingredients_list}}
+
+# Or vector search if embeddings are pre-computed
+{"ingredients_vector": {"$near": query_embedding}}
+```
+
+## Data Preparation
+
+### Ingredient Processing Rules
+1. **Clean individual items**: "2 cups rice" → "rice"  
+2. **Remove measurements**: "1 lb chicken breast" → "chicken breast"
+3. **Lowercase**: "Fresh Basil" → "fresh basil"
+4. **Array format**: ["rice", "chicken breast", "fresh basil"]
+5. **No duplicates**: Remove duplicate ingredients from array
+
+### Example Transformation
+```
+Raw: "2 lbs fresh shrimp, 1 cup jasmine rice (cooked), 1/2 cup coconut milk"
+Clean: ["fresh shrimp", "jasmine rice", "coconut milk"]
+```
+
+## Benefits
+
+### 1. Simplicity
+- Single structure for all providers
+- Easy to understand and maintain
+- No complex transformations needed
+
+### 2. Performance  
+- Optimized for ingredient matching
+- Fast text and vector search
+- Minimal processing overhead
+
+### 3. Flexibility
+- Works with existing MongoDB data
+- Compatible with ChromaDB auto-embedding
+- Supports both search types (text/vector)
+
+### 4. Scalability
+- Easy to add new recipes
+- Simple data validation
+- Consistent across providers
+
+This universal structure ensures maximum compatibility and efficiency for ingredient-based recipe recommendations across all vector store providers.

backend/docs/sanitization_guide.md

@@ -0,0 +1,147 @@
+# Simplified Data Sanitization Documentation
+
+## Overview
+
+The simplified data sanitization module provides focused input validation and sanitization for the Recipe Recommendation Bot API. It's designed specifically for recipe chatbot context with essential security protection.
+
+## Features
+
+### 🛡️ **Essential Security Protection**
+- **XSS Prevention**: HTML encoding and basic script removal
+- **Input Validation**: Length limits and content validation
+- **Whitespace Normalization**: Clean formatting
+
+### 🔧 **Simple Configuration**
+- **Maximum Message Length**: 1000 characters
+- **Minimum Message Length**: 1 character
+- **Single Method**: One sanitization method for all inputs
+
+## Usage
+
+### Basic Sanitization
+
+```python
+from utils.sanitization import sanitize_user_input
+
+# Sanitize any user input (chat messages, demo prompts)
+clean_input = sanitize_user_input("What are some chicken recipes?")
+```
+
+### Advanced Usage
+
+```python
+from utils.sanitization import DataSanitizer
+
+# Direct class usage
+sanitizer = DataSanitizer()
+clean_text = sanitizer.sanitize_input("User input")
+```
+
+## Security Patterns Handled
+
+### Basic XSS Protection
+- `<script>` tags → Removed
+- `javascript:` URLs → Cleaned
+- Event handlers (`onclick`, `onload`) → Removed
+- HTML entities → Properly encoded
+
+### Input Validation
+- Length limits (1-1000 characters)
+- Empty input detection
+- Whitespace normalization
+
+## Integration
+
+The sanitization is automatically applied in FastAPI endpoints:
+
+### Chat Endpoint
+```python
+class ChatMessage(BaseModel):
+    message: str = Field(..., min_length=1, max_length=1000)
+    
+    @validator('message')
+    def sanitize_message_field(cls, v):
+        return sanitize_user_input(v)
+```
+
+### Demo Endpoint
+```python
+@app.get("/demo")
+def demo(prompt: str = "What recipes do you have?"):
+    sanitized_prompt = sanitize_user_input(prompt)
+    # ... rest of the logic
+```
+
+## Error Handling
+
+The sanitization raises `ValueError` for invalid input:
+
+```python
+try:
+    clean_input = sanitize_user_input(user_input)
+except ValueError as e:
+    return {"error": f"Invalid input: {str(e)}"}
+```
+
+## Testing
+
+Run the sanitization tests:
+
+```bash
+python3 test_sanitization.py
+```
+
+The test suite covers:
+- Normal recipe-related messages
+- Basic harmful content (scripts, JavaScript)
+- Length validation
+- Whitespace normalization
+- Edge cases
+
+## What's Simplified
+
+### Removed Overly Complex Features:
+- ❌ SQL injection patterns (not relevant for LLM chatbot)
+- ❌ Command injection patterns (not applicable)
+- ❌ Separate strict/relaxed modes (unnecessary complexity)
+- ❌ Multiple sanitization methods (unified approach)
+
+### Kept Essential Features:
+- ✅ Basic XSS protection
+- ✅ Input length validation
+- ✅ HTML encoding
+- ✅ Whitespace normalization
+- ✅ Clear error messages
+
+## Performance
+
+- **Lightweight**: Minimal regex patterns
+- **Fast**: Simple operations only
+- **Memory Efficient**: No complex state
+- **Recipe-Focused**: Context-appropriate validation
+
+## Examples
+
+### Valid Inputs (Cleaned):
+```python
+"What are chicken recipes?" → "What are chicken recipes?"
+"<script>alert('xss')</script>Tell me about pasta" → "Tell me about pasta"
+"   How to cook rice?   " → "How to cook rice?"
+"What about desserts & sweets?" → "What about desserts &amp; sweets?"
+```
+
+### Invalid Inputs (Rejected):
+```python
+"" → ValueError: Input cannot be empty
+"a" * 1001 → ValueError: Input too long (maximum 1000 characters)
+```
+
+## Best Practices
+
+1. **Keep It Simple**: Focus on actual threats for recipe chatbot
+2. **Context Appropriate**: Don't over-engineer for non-existent threats
+3. **User Friendly**: Allow normal recipe-related punctuation
+4. **Clear Errors**: Provide helpful error messages
+5. **Test Regularly**: Verify with real recipe queries
+
+This simplified approach provides adequate protection while maintaining usability for a recipe recommendation chatbot context.

backend/docs/scraper.md

@@ -0,0 +1,372 @@
+# Recipe Scraper – FastAPI demo
+
+A tiny FastAPI service + CLI that scrapes recipe sites, normalizes data, and (optionally) embeds combined **ingredients + instructions** into a single vector (`recipe_emb`). Designed as a **test project**—simple to run locally, easy to extend.
+
+---
+
+## Features
+
+* 🔧 **Sites**: `yummy` (YummyMedley), `anr` (All Nigerian Recipes)
+* 🧱 **Unified text**: builds `recipe_text` from sections, or embeds `("ingredients","instructions") → recipe_emb`
+* 🧠 **Embeddings**: Hugging Face `sentence-transformers` via your `HFEmbedder` (default: `all-MiniLM-L6-v2`)
+* 🚀 **API trigger**: `POST /scrape` runs scraping in the background
+* 👀 **Progress**: `GET /jobs/{job_id}` (and optional `GET /jobs`) to check status
+* 💾 **Output**: `output_type = "json"` (local file) or `"mongo"` (MongoDB/Atlas)
+
+---
+
+## Project layout (essential bits)
+
+```
+backend/
+  app.py
+  data_minning/
+      base_scraper.py       # BaseRecipeScraper (+ StreamOptions)
+      all_nigerian_recipe_scraper.py
+      yummy_medley_scraper.py
+      dto/recipe_doc.py
+      soup_client.py
+  utils/sanitization.py
+```
+
+Make sure every package dir has an `__init__.py`.
+
+---
+
+## Requirements
+
+* Python 3.9+
+* macOS/Linux (Windows should work too)
+* (Optional) MongoDB/Atlas for `"mongo"` output
+
+### Install
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+
+pip install --upgrade pip
+pip install -r requirements.txt
+# If you don’t have a requirements.txt, minimum:
+pip install fastapi "uvicorn[standard]" pydantic==2.* requests beautifulsoup4 \
+            sentence-transformers numpy pymongo python-dotenv
+```
+
+> If `uvicorn` isn’t found on your PATH, you can always run with `python3 -m uvicorn ...`.
+
+---
+
+## Environment variables
+
+Create `.env` in repo root (or export envs) as needed:
+
+```dotenv
+
+
+# For Mongo output_type="mongo"
+MONGODB_URI=mongodb+srv://user:pass@cluster/recipes?retryWrites=true&w=majority
+MONGODB_DB=recipes
+MONGODB_COL=items
+ATLAS_INDEX=recipes_vec  # your Atlas Search index name
+
+# Embeddings (HFEmbedder)
+HF_MODEL=sentence-transformers/all-MiniLM-L6-v2
+HF_DEVICE=cpu  # or cuda
+```
+
+---
+
+## Running the API
+
+From the project root (the folder **containing** `backend/`):
+
+```bash
+python3 -m uvicorn app:app --reload --host 127.0.0.1 --port 8080
+```
+
+
+---
+
+## API
+
+### POST `/scrape`
+
+Trigger a scrape job (non-blocking). **Body** is a JSON object:
+
+```json
+{
+  "site": "yummy",
+  "limit": 50, #optional
+  "output_type": "json"   // or "mongo"
+}
+```
+
+**Headers**
+
+* `Content-Type: application/json`
+* If enabled: `X-API-Key: <ADMIN_API_KEY>`
+
+**curl example (JSON output):**
+
+```bash
+curl -X POST http://127.0.0.1:8080/scrape \
+  -H "Content-Type: application/json" \
+  -H "X-API-Key: dev-key" \
+  -d '{"site":"yummy","limit":20,"output_type":"json"}'
+```
+
+**Response**
+
+```json
+{ "job_id": "yummy-a1b2c3d4", "status": "queued" }
+```
+
+### GET `/jobs/{job_id}`
+
+Check progress:
+
+```bash
+curl http://127.0.0.1:8080/jobs/yummy-a1b2c3d4
+```
+
+**Possible responses**
+
+```json
+{ "status": "running", "count": 13 }
+{ "status": "done", "count": 50 }
+{ "status": "error", "error": "Traceback ..." }
+{ "status": "unknown" }
+```
+
+### (Optional) GET `/jobs`
+
+Return the whole in-memory job map (useful for debugging):
+
+```bash
+curl http://127.0.0.1:8080/jobs
+```
+
+> Note: jobs are stored in a process-local dict and clear on server restart.
+
+---
+
+## Output modes
+
+### `"json"`
+
+Writes batches to a JSON sink (e.g., newline-delimited file). Check the sink path configured in your `JsonArraySink`/`DualSink`.
+
+Typical document shape:
+
+```json
+{
+  "title": "...",
+  "url": "...",
+  "source": "...",
+  "category": "...",
+  "ingredients": "- 1 cup rice\n- 2 tbsp oil\n...",
+  "instructions": "1. Heat oil...\n\n2. Add rice...",
+  "image_url": "...",
+  "needs_review": false,
+  "scraped_at": "2025-09-14 10:03:32.289232",
+  "recipe_emb": [0.0123, -0.0456, ...]   // when embeddings enabled
+}
+```
+
+### `"mongo"`
+
+Writes to `MONGODB_DB.MONGODB_COL`. Ensure your Atlas Search index is created if you plan to query vectors.
+
+**Atlas index mapping (single vector field)**
+
+```json
+{
+  "mappings": {
+    "dynamic": false,
+    "fields": {
+      "recipe_emb": { "type": "knnVector", "dims": 384, "similarity": "cosine" }
+    }
+  }
+}
+```
+
+**Query example:**
+
+```python
+qvec = embedder.encode([query])[0]
+pipeline = [{
+  "$vectorSearch": {
+    "index": os.getenv("ATLAS_INDEX", "recipes_vec"),
+    "path": "recipe_emb",
+    "queryVector": qvec,
+    "numCandidates": 400,
+    "limit": 10,
+    "filter": { "needs_review": { "$ne": True } }
+  }
+}]
+results = list(col.aggregate(pipeline))
+```
+
+---
+
+## Embeddings (combined fields → one vector)
+
+We embed **ingredients + instructions** into a single `recipe_emb`. Two supported patterns:
+
+### A) Combine at embedding time
+
+Configure:
+
+```python
+embedding_fields = [
+  (("ingredients", "instructions"), "recipe_emb")
+]
+```
+
+`_apply_embeddings` concatenates labeled sections:
+
+```
+Ingredients:
+- ...
+
+Instructions:
+1. ...
+```
+
+### B) Build `recipe_text` in `RecipeDoc.finalize()` and embed once
+
+```python
+self.recipe_text = "\n\n".join(
+  [s for s in [
+    f"Title:\n{self.title}" if self.title else "",
+    f"Ingredients:\n{self.ingredients_text}" if self.ingredients_text else "",
+    f"Instructions:\n{self.instructions_text}" if self.instructions_text else ""
+  ] if s]
+)
+# embedding_fields = [("recipe_text", "recipe_emb")]
+```
+
+**HFEmbedder config (defaults):**
+
+```python
+HF_MODEL=sentence-transformers/all-MiniLM-L6-v2
+HF_DEVICE=cpu
+```
+
+---
+
+## CLI (optional but handy)
+
+Create `run_scrape.py`:
+
+```python
+from backend.services.data_minning.yummy_medley_scraper import YummyMedleyScraper
+from backend.services.data_minning.all_nigerian_recipe_scraper import AllNigerianRecipesScraper
+
+SCRAPERS = {
+  "yummy": YummyMedleyScraper,
+  "anr": AllNigerianRecipesScraper,
+}
+
+if __name__ == "__main__":
+    import argparse
+    from dataclasses import asdict
+    p = argparse.ArgumentParser()
+    p.add_argument("--site", choices=SCRAPERS.keys(), required=True)
+    p.add_argument("--limit", type=int, default=50)
+    args = p.parse_args()
+
+    s = SCRAPERS[args.site]()
+    saved = s.stream(sink=..., options=StreamOptions(limit=args.limit))
+    print(f"Saved {saved}")
+```
+
+Run:
+
+```bash
+python3 run_scrape.py --site yummy --limit 25
+```
+
+---
+
+## Implementation notes
+
+### `StreamOptions` (clean params)
+
+```python
+from dataclasses import dataclass
+from typing import Optional, Callable
+
+@dataclass
+class StreamOptions:
+    delay: float = 0.3
+    limit: Optional[int] = None
+    batch_size: int = 50
+    resume_file: Optional[str] = None
+    progress_callback: Optional[Callable[[int], None]] = None
+```
+
+### Progress to `/jobs`
+
+We pass a `progress_callback` that updates the job by `job_id`:
+
+```python
+def make_progress_cb(job_id: str):
+    def _cb(n: int):
+        JOBS[job_id]["count"] = n
+    return _cb
+```
+
+Used as:
+
+```python
+saved = s.stream(
+  sink=json_or_mongo_sink,
+  options=StreamOptions(
+    limit=body.limit,
+    batch_size=body.limit,
+    resume_file="recipes.resume",
+    progress_callback=make_progress_cb(job_id),
+  ),
+)
+```
+
+---
+
+## Common pitfalls & fixes
+
+* **`ModuleNotFoundError: No module named 'backend'`**
+  Run with module path:
+  `python3 -m uvicorn backend.app:app --reload`
+
+* **Uvicorn not found (`zsh: command not found: uvicorn`)**
+  Use: `python3 -m uvicorn ...` or add `~/Library/Python/3.9/bin` to PATH.
+
+* **`422 Unprocessable Entity` on `/scrape`**
+  In Postman: Body → **raw → JSON** and send:
+  `{"site":"yummy","limit":20,"output_type":"json"}`
+
+* **Pydantic v2: “non-annotated attribute”**
+  Keep globals like `JOBS = {}` **outside** `BaseModel` classes.
+
+* **`'int' object is not iterable`**
+  Don’t iterate `stream()`—it **returns** an `int`. Use the `progress_callback` if you need live updates.
+
+* **`BackgroundTasks` undefined**
+  Import from FastAPI:
+  `from fastapi import BackgroundTasks`
+
+* **Too many commas in ingredients**
+  Don’t `.join()` a **string**—only join if it’s a `list[str]`.
+
+---
+
+## Future ideas (nice-to-haves)
+
+* Store jobs in Redis for persistence across restarts
+* Add `started_at` / `finished_at` timestamps and durations to jobs
+* Rate-limit per site; cool-down if a scrape ran recently
+* Switch to task queue (Celery/RQ/BullMQ) if you need scale
+* Add `/search` endpoint that calls `$vectorSearch` in MongoDB
+
+---

backend/docs/unified-provider-configuration.md

@@ -0,0 +1,108 @@
+# Unified Provider Configuration
+
+## Overview
+The Recipe Recommendation Bot now uses a **unified provider approach** where a single `LLM_PROVIDER` setting controls both LLM and embedding models. This eliminates configuration mismatches and simplifies setup.
+
+## Before vs After
+
+### ❌ Previous Approach (Confusing)
+```bash
+LLM_PROVIDER=huggingface
+EMBEDDING_PROVIDER=openai  # 😵 Different providers - causes issues!
+```
+
+### ✅ New Approach (Simple)
+```bash
+LLM_PROVIDER=huggingface  # 🎯 One setting controls both LLM and embeddings
+```
+
+## Benefits
+
+1. **Prevents Mismatches**: No more accidentally mixing providers
+2. **Simplified Configuration**: One setting instead of two
+3. **Better User Experience**: Less confusion, fewer errors
+4. **Consistent Performance**: Same provider for both LLM and embeddings
+5. **Easier Troubleshooting**: Single provider to debug
+
+## Supported Combinations
+
+| Provider | LLM Model | Embedding Model |
+|----------|-----------|-----------------|
+| `openai` | `gpt-5-nano` | `text-embedding-3-small` |
+| `google` | `gemini-2.0-flash` | `models/embedding-001` |
+| `huggingface` | `microsoft/DialoGPT-small` | `sentence-transformers/all-MiniLM-L6-v2` |
+
+## Configuration Examples
+
+### OpenAI (Complete Setup)
+```bash
+LLM_PROVIDER=openai
+OPENAI_API_KEY=your_api_key_here
+OPENAI_MODEL=gpt-5-nano
+OPENAI_EMBEDDING_MODEL=text-embedding-3-small
+```
+
+### Google (Complete Setup)
+```bash
+LLM_PROVIDER=google
+GOOGLE_API_KEY=your_api_key_here
+GOOGLE_MODEL=gemini-2.0-flash
+GOOGLE_EMBEDDING_MODEL=models/embedding-001
+```
+
+### HuggingFace (Complete Setup)
+```bash
+LLM_PROVIDER=huggingface
+HUGGINGFACE_API_TOKEN=your_token_here
+HUGGINGFACE_MODEL=microsoft/DialoGPT-small
+HUGGINGFACE_USE_GPU=false
+HUGGINGFACE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+```
+
+## Migration Guide
+
+If you have an existing `.env` file:
+
+1. **Remove** the `EMBEDDING_PROVIDER` line
+2. **Keep** the `LLM_PROVIDER` line
+3. **Ensure** both LLM and embedding model settings are configured for your chosen provider
+
+### Example Migration
+```bash
+# OLD .env
+LLM_PROVIDER=huggingface
+EMBEDDING_PROVIDER=huggingface  # ← Remove this line
+
+# NEW .env
+LLM_PROVIDER=huggingface  # ← Keep this, it controls both
+```
+
+## Technical Implementation
+
+The configuration system now:
+- Uses `LLM_PROVIDER` for both `get_llm_config()` and `get_embedding_config()`
+- Automatically matches provider types
+- Validates that the provider supports both LLM and embeddings
+- Provides clear error messages for unsupported providers
+
+## Validation
+
+You can verify your configuration works:
+```bash
+cd backend
+python -c "
+from config.settings import settings
+llm = settings.get_llm_config()
+emb = settings.get_embedding_config()
+print(f'LLM: {llm[\"provider\"]}')
+print(f'Embedding: {emb[\"provider\"]}')
+print(f'Match: {llm[\"provider\"] == emb[\"provider\"]}')
+"
+```
+
+Expected output:
+```
+LLM: huggingface
+Embedding: huggingface
+Match: True
+```

backend/requirements.txt

@@ -0,0 +1,49 @@
+# Production Requirements - Core dependencies only
+# Keep this minimal for production deployments
+
+# Core API dependencies
+fastapi
+uvicorn[standard]
+python-dotenv
+python-multipart
+
+# Data processing
+numpy
+requests
+
+# LLM Providers
+openai
+google-generativeai
+
+# Vector Store & Embeddings (optional - choose based on needs)
+# Uncomment if needed in production
+chromadb
+langchain-mongodb
+pymongo
+
+# MongoDB Atlas Vector Store (optional)
+pymongo[srv]
+
+# HuggingFace dependencies
+# transformers
+# accelerate
+
+# Sentence Transformers - Choose ONE option below:
+# FULL sentence-transformers (easier to use, ~800MB+ )
+# sentence-transformers
+
+# Note: sentence-transformers will automatically use CPU-only PyTorch if CUDA is not available (easier to use, ~200MB - 300MB )
+# To force CPU-only installation: pip install torch --index-url https://download.pytorch.org/whl/cpu && pip install sentence-transformers
+
+# LangChain for RAG
+langchain
+langchain-core
+langchain-text-splitters
+langchain-openai
+langchain-chroma
+langchain_google_genai
+langchain-huggingface
+langchain-community
+
+bs4
+lxml

backend/services/__init__.py

@@ -0,0 +1 @@
+# Services package initialization

backend/services/custom_mongo_vector.py

@@ -0,0 +1,154 @@
+"""
+Streamlined MongoDB Vector Store with Atlas Vector Search
+"""
+
+from typing import List, Dict, Any, Optional, NamedTuple
+import numpy as np
+from langchain.schema import Document
+from langchain.vectorstores.base import VectorStore
+from pymongo.collection import Collection
+from backend.config.logging_config import get_logger
+
+logger = get_logger("custom_mongo_vector")
+
+class VectorSearchOptions(NamedTuple):
+    """Configuration options for vector search"""
+    index_name: str = "foodInstructionIndex"
+    embedding_key: str = "ingredients_emb"
+    text_key: str = "title"
+    num_candidates: int = 50
+    similarity_metric: str = "cosine"  # cosine or dotProduct
+
+class CustomMongoDBVectorStore(VectorStore):
+    """
+    Streamlined MongoDB Atlas Vector Store with efficient $vectorSearch aggregation.
+    Falls back to Python similarity calculation when Atlas Vector Search is unavailable.
+    """
+    
+    def __init__(
+        self,
+        collection: Collection,
+        embedding_function,
+        options: Optional[VectorSearchOptions] = None
+    ):
+        self.collection = collection
+        self.embedding_function = embedding_function
+        self.options = options or VectorSearchOptions()
+        
+        logger.info(f"🔧 Streamlined MongoDB Vector Store initialized")
+        logger.info(f"� Config: {self.options.index_name} index, {self.options.similarity_metric} similarity")
+    
+    def _calculate_similarity(self, vec1: List[float], vec2: List[float]) -> float:
+        """Calculate similarity using the most efficient method"""
+        a, b = np.array(vec1), np.array(vec2)
+        
+        if self.options.similarity_metric == "dotProduct":
+            # Dot product (faster, good for normalized embeddings)
+            return float(np.dot(a, b))
+        else:
+            # Cosine similarity (more robust, handles non-normalized embeddings)
+            return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))
+    
+    def similarity_search(self, query: str, k: int = 4, **kwargs: Any) -> List[Document]:
+        """Streamlined similarity search using Atlas Vector Search with Python fallback"""
+        logger.info(f"🔍 Searching: '{query}' (k={k})")
+        
+        qvec = self.embedding_function.embed_query(query)
+        
+        # Primary: Try Atlas Vector Search (efficient, server-side)
+        try:
+            pipeline = [
+                {
+                    "$vectorSearch": {
+                        "index": self.options.index_name,
+                        "path": self.options.embedding_key,
+                        "queryVector": qvec,
+                        "numCandidates": self.options.num_candidates,
+                        "limit": k
+                    }
+                },
+                {
+                    "$match": {
+                        '$or': [
+                            { 'needs_review': { '$exists': False } },
+                            { 'needs_review': False }
+                        ]
+                    }
+                }
+            ]
+            
+            results = list(self.collection.aggregate(pipeline))
+            if results:
+                logger.info(f"✅ Atlas Vector Search: {len(results)} results")
+                return self._create_documents(results)
+                
+        except Exception as e:
+            logger.warning(f"⚠️ Atlas Vector Search failed: {e}")
+        
+        # Fallback: Python similarity calculation
+        logger.info("🔄 Using Python similarity fallback")
+        return self._python_similarity_search(qvec, k)
+    
+    def _python_similarity_search(self, qvec: List[float], k: int) -> List[Document]:
+        """Efficient Python-based similarity search fallback"""
+        cursor = self.collection.find(
+            {'$or': [
+                {'needs_review': {'$exists': False}},
+                {'needs_review': False}
+            ]},
+            {self.options.text_key: 1, self.options.embedding_key: 1, "ingredients": 1, "instructions": 1}
+        )
+        
+        # Vectorized similarity calculation for efficiency
+        similarities = []
+        for doc in cursor:
+            doc_emb = doc.get(self.options.embedding_key)
+            if doc_emb and len(doc_emb) == len(qvec):
+                score = self._calculate_similarity(qvec, doc_emb)
+                similarities.append((doc, score))
+        
+        # Return top-k results
+        similarities.sort(key=lambda x: x[1], reverse=True)
+        top_docs = [doc for doc, _ in similarities[:k]]
+        
+        logger.info(f"📊 Python fallback: {len(similarities)} processed, {len(top_docs)} returned")
+        return self._create_documents(top_docs)
+    
+    def _create_documents(self, docs: List[Dict]) -> List[Document]:
+        """Create LangChain Documents from MongoDB results using clean string content"""
+        documents = []
+        for doc in docs:
+            title = doc.get(self.options.text_key, "Untitled Recipe")
+            ingredients = doc.get("ingredients", "")
+            instructions = doc.get("instructions", "")
+            
+            # Build clean content without extra formatting
+            content_parts = [f"Recipe: {title}"]
+            
+            if ingredients:
+                content_parts.append(f"Ingredients: {ingredients}")
+            
+            if instructions:
+                content_parts.append(f"Instructions: {instructions}")
+            
+            content = "\n\n".join(content_parts)
+            
+            documents.append(Document(
+                page_content=content,
+                metadata={"_id": str(doc["_id"]), "title": title}
+            ))
+        
+        return documents
+    
+    def similarity_search_with_score(self, query: str, k: int = 4, **kwargs: Any) -> List[tuple]:
+        """Return docs with similarity scores (simplified)"""
+        docs = self.similarity_search(query, k, **kwargs)
+        return [(doc, 1.0) for doc in docs]  # Atlas Vector Search doesn't return raw scores
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]] = None, **kwargs: Any) -> List[str]:
+        """Read-only vector store - adding texts not supported"""
+        raise NotImplementedError("This vector store is read-only for pre-existing embeddings")
+    
+    @classmethod
+    def from_texts(cls, texts: List[str], embedding_function, metadatas: Optional[List[dict]] = None, **kwargs: Any):
+        """Read-only vector store - creating from texts not supported"""
+        raise NotImplementedError("This vector store is read-only for pre-existing embeddings")

backend/services/llm_service.py

@@ -0,0 +1,354 @@
+# LLM Service - RAG pipeline using ConversationalRetrievalChain
+from typing import List, Dict, Any, Optional
+
+# Local imports
+from backend.config.settings import settings
+from backend.config.logging_config import get_logger
+from backend.services.vector_store import vector_store_service
+
+# Setup logging
+logger = get_logger("llm_service")
+
+class LLMService:
+    """LLM service using ConversationalRetrievalChain for RAG pipeline"""
+    
+    def __init__(self):
+        logger.info("🤖 Initializing LLM Service...")
+        
+        try:
+            self.llm = self._setup_llm()
+            self.retriever = self._setup_retriever()
+            self.memory = self._setup_memory()
+            self.qa_chain = self._setup_qa_chain()
+            
+            logger.info("🚀 LLM Service initialized successfully")
+            
+        except Exception as e:
+            logger.error(f"❌ LLM Service initialization failed: {str(e)}", exc_info=True)
+            raise
+    
+    def _setup_llm(self):
+        """Setup LLM based on configuration with conditional imports"""
+        llm_config = settings.get_llm_config()
+        provider = llm_config["provider"]
+        
+        logger.info(f"🔧 Setting up LLM provider: {provider}")
+        
+        if provider == "openai":
+            try:
+                from langchain_openai import ChatOpenAI
+                logger.info("✅ OpenAI LLM imported successfully")
+                
+                # Handle special cases for temperature restrictions
+                temperature = llm_config["temperature"]
+                model = llm_config["model"]
+                max_tokens = llm_config.get("max_tokens", 1000)
+                
+                # GPT-5-nano has temperature restrictions (defaults to 1.0)
+                if "gpt-5-nano" in model.lower():
+                    temperature = 1.0
+                    logger.info(f"🔧 Using temperature=1.0 for {model} (model restriction)")
+                
+                # Log token configuration
+                logger.info(f"🎯 OpenAI config - Model: {model}, Output tokens: {max_tokens}, Temperature: {temperature}")
+                
+                return ChatOpenAI(
+                    api_key=llm_config["api_key"],
+                    model=model,
+                    temperature=temperature,
+                    max_tokens=max_tokens  # This limits OUTPUT tokens only
+                )
+            except ImportError as e:
+                logger.error(f"❌ OpenAI LLM not available: {e}")
+                raise ImportError("OpenAI provider selected but langchain_openai not installed")
+        
+        elif provider == "google":
+            try:
+                from langchain_google_genai import ChatGoogleGenerativeAI
+                logger.info("✅ Google LLM imported successfully")
+                
+                max_output_tokens = llm_config.get("max_tokens", 1000)
+                model = llm_config["model"]
+                temperature = llm_config["temperature"]
+                
+                # Log token configuration
+                logger.info(f"🎯 Google config - Model: {model}, Output tokens: {max_output_tokens}, Temperature: {temperature}")
+                
+                return ChatGoogleGenerativeAI(
+                    google_api_key=llm_config["api_key"],
+                    model=model,
+                    temperature=temperature,
+                    max_output_tokens=max_output_tokens  # This limits OUTPUT tokens only
+                )
+            except ImportError as e:
+                logger.error(f"❌ Google LLM not available: {e}")
+                raise ImportError("Google provider selected but langchain_google_genai not installed")
+        
+        elif provider == "ollama":
+            try:
+                from langchain_community.llms import Ollama
+                logger.info("✅ Ollama LLM imported successfully")
+                return Ollama(
+                    base_url=llm_config["base_url"],
+                    model=llm_config["model"],
+                    temperature=llm_config["temperature"]
+                )
+            except ImportError as e:
+                logger.error(f"❌ Ollama LLM not available: {e}")
+                raise ImportError("Ollama provider selected but langchain_community not installed")
+        
+        elif provider == "huggingface":
+            try:
+                # Check if we should use API or local pipeline
+                use_api = llm_config.get("use_api", False)
+                
+                if use_api:
+                    # Use HuggingFace Inference API with better error handling
+                    try:
+                        from langchain_huggingface import HuggingFaceEndpoint
+                        logger.info("✅ Using HuggingFace API (no local download)")
+                        
+                        return HuggingFaceEndpoint(
+                            repo_id=llm_config["model"],
+                            huggingfacehub_api_token=llm_config["api_token"],
+                            temperature=0.7,  # HuggingFace API doesn't support dynamic temperature from config
+                            max_new_tokens=200,
+                            repetition_penalty=1.1,
+                            top_p=0.9
+                        )
+                    except Exception as api_error:
+                        logger.warning(f"⚠️ HuggingFace API failed: {api_error}")
+                        logger.info("🔄 Falling back to HuggingFace Hub API...")
+                        
+                        # Fallback to HuggingFaceHub (older but more reliable)
+                        try:
+                            from langchain_community.llms import HuggingFaceHub
+                            
+                            return HuggingFaceHub(
+                                repo_id=llm_config["model"],
+                                huggingfacehub_api_token=llm_config["api_token"],
+                                model_kwargs={
+                                    "temperature": 0.7,  # HuggingFace Hub API has limited temperature control
+                                    "max_new_tokens": 200,
+                                    "repetition_penalty": 1.1,
+                                    "top_p": 0.9,
+                                    "do_sample": True
+                                }
+                            )
+                        except Exception as hub_error:
+                            logger.error(f"❌ HuggingFace Hub also failed: {hub_error}")
+                            raise ImportError(f"Both HuggingFace API methods failed: {api_error}, {hub_error}")
+                else:
+                    # Use local pipeline (downloads model)
+                    from langchain_huggingface import HuggingFacePipeline
+                    from transformers import pipeline
+                    
+                    logger.info("✅ Using HuggingFace local pipeline")
+                    
+                    # Create HuggingFace pipeline - avoid device_map for CPU-only setups
+                    pipeline_kwargs = {
+                        "task": "text-generation",
+                        "model": llm_config["model"],
+                        "max_length": 512,  # Increase max length
+                        "do_sample": True,  # Enable sampling for better responses
+                        "temperature": 0.7,  # Local pipeline uses default 0.7 for stability
+                        "pad_token_id": 50256,  # Set pad token to avoid warnings
+                        "eos_token_id": 50256,  # Set end of sequence token
+                    }
+                
+                    # Only add device_map if using GPU
+                    if llm_config.get("use_gpu", False):
+                        pipeline_kwargs["device_map"] = "auto"
+                    else:
+                        # For CPU, use device=0 which maps to CPU
+                        pipeline_kwargs["device"] = "cpu"
+                    
+                    hf_pipeline = pipeline(**pipeline_kwargs)
+                    
+                    return HuggingFacePipeline(
+                        pipeline=hf_pipeline,
+                        model_kwargs={
+                            "temperature": 0.7,  # Local pipeline temperature (limited configurability)
+                            "max_new_tokens": 150,  # Reduced for efficiency
+                            "do_sample": True,
+                            "top_p": 0.9,
+                            "repetition_penalty": 1.1,
+                            "early_stopping": True,
+                            "num_beams": 4  # Better quality for instruction following
+                        }
+                    )
+            except ImportError as e:
+                logger.error(f"❌ HuggingFace LLM not available: {e}")
+                raise ImportError("HuggingFace provider selected but required packages not installed")
+        
+        else:
+            logger.warning(f"⚠️ Unknown LLM provider '{provider}', falling back to OpenAI")
+            try:
+                from langchain_openai import ChatOpenAI
+                return ChatOpenAI()
+            except ImportError:
+                logger.error("❌ No valid LLM provider available")
+                raise ImportError("No valid LLM provider available")
+    
+    def _setup_retriever(self):
+        """Setup retriever from vector store service"""
+        return vector_store_service.get_retriever()
+    
+    def _setup_memory(self):
+        """Setup conversation memory"""
+        try:
+            from langchain.memory import ConversationBufferMemory
+            return ConversationBufferMemory(memory_key='chat_history', return_messages=True)
+        except ImportError as e:
+            logger.error(f"❌ ConversationBufferMemory not available: {e}")
+            raise ImportError("langchain memory not available")
+    
+    def _setup_qa_chain(self):
+        """Setup ConversationalRetrievalChain"""
+        try:
+            from langchain.chains import ConversationalRetrievalChain
+            return ConversationalRetrievalChain.from_llm(
+                llm=self.llm,
+                retriever=self.retriever,
+                memory=self.memory,
+                verbose=settings.LANGCHAIN_DEBUG  # Reduce debugging noise
+            )
+        except ImportError as e:
+            logger.error(f"❌ ConversationalRetrievalChain not available: {e}")
+            raise ImportError("langchain chains not available")
+    
+    def _preprocess_query(self, question: str) -> str:
+        """Preprocess user query to improve vector search accuracy"""
+        import re
+        
+        # Convert to lowercase for consistency
+        processed = question.lower()
+        
+        # Remove common stop words that don't help with recipe matching
+        stop_words = ['i', 'want', 'a', 'an', 'the', 'for', 'with', 'can', 'you', 'give', 'me', 'please', 'help']
+        words = processed.split()
+        words = [word for word in words if word not in stop_words]
+        
+        # Remove punctuation except spaces
+        processed = ' '.join(words)
+        processed = re.sub(r'[^\w\s]', '', processed)
+        
+        # Normalize multiple spaces
+        processed = ' '.join(processed.split())
+        
+        logger.debug(f"🔧 Query preprocessing: '{question}' → '{processed}'")
+        return processed
+
+    def ask_question(self, user_question: str) -> str:
+        """Ask a question using the conversational retrieval chain"""
+        logger.info(f"❓ Processing: '{user_question[:60]}...'")
+        
+        try:
+            # Preprocess query for better matching
+            processed_query = self._preprocess_query(user_question)
+            
+            # Get context for token tracking
+            document_retriever = getattr(self.qa_chain, 'retriever', None)
+            retrieved_context = ""
+            if document_retriever:
+                # Use both queries for comprehensive results
+                original_docs = document_retriever.invoke(user_question)
+                processed_docs = document_retriever.invoke(processed_query)
+                
+                # Deduplicate documents
+                seen_content = set()
+                unique_documents = []
+                for document in original_docs + processed_docs:
+                    if document.page_content not in seen_content:
+                        unique_documents.append(document)
+                        seen_content.add(document.page_content)
+                
+                retrieved_context = "\n".join([doc.page_content for doc in unique_documents[:8]])
+                logger.debug(f"📄 Retrieved {len(unique_documents)} unique documents")
+            
+            # Enhanced question for natural responses
+            enhanced_question = f"""Based on the available recipe information, please answer this cooking question: "{user_question}"
+
+Respond directly and naturally as if you're sharing your own culinary knowledge. If there's a specific recipe that matches the request, share the complete recipe with ingredients and step-by-step instructions in a friendly, conversational way."""
+            
+            result = self.qa_chain({"question": enhanced_question})
+            generated_answer = result["answer"]
+            
+            self._log_token_usage(user_question, retrieved_context, generated_answer)
+            
+            logger.info(f"✅ Response generated ({len(generated_answer)} chars)")
+            return generated_answer
+                
+        except Exception as error:
+            logger.error(f"❌ Error in ask_question: {str(error)}")
+            return f"Sorry, I encountered an error: {str(error)}"
+    
+    def _count_tokens(self, text: str) -> int:
+        """Count tokens in text (rough estimate for debugging)"""
+        return len(text) // 4 if text else 0
+    
+    def _log_token_usage(self, question: str, context: str, response: str):
+        """Log token usage for monitoring"""
+        question_tokens = self._count_tokens(question)
+        context_tokens = self._count_tokens(context)
+        response_tokens = self._count_tokens(response)
+        total_input_tokens = question_tokens + context_tokens
+        
+        logger.info(f"📊 Token Usage - Input:{total_input_tokens} (Q:{question_tokens}+C:{context_tokens}), Output:{response_tokens}")
+        
+        if context_tokens > 3000:
+            logger.warning(f"⚠️ Large context detected: {context_tokens} tokens")
+            
+        return {
+            "input_tokens": total_input_tokens, 
+            "output_tokens": response_tokens, 
+            "total_tokens": total_input_tokens + response_tokens
+        }
+    
+    def clear_memory(self):
+        """Clear conversation memory"""
+        try:
+            if hasattr(self.memory, 'clear'):
+                self.memory.clear()
+                logger.info("✅ Memory cleared")
+                return True
+        except Exception as e:
+            logger.warning(f"⚠️ Could not clear memory: {e}")
+        return False
+
+    def simple_chat_completion(self, user_message: str) -> str:
+        """Simple chat completion without RAG - direct LLM response"""
+        logger.info(f"💭 Simple chat: '{user_message[:50]}...'")
+        
+        try:
+            llm_prompt = f"As a knowledgeable cooking expert, share your insights about {user_message}. Provide helpful culinary advice and recommendations:\n\n"
+            
+            llm_response = self.llm.invoke(llm_prompt) if hasattr(self.llm, 'invoke') else self.llm(llm_prompt)
+            
+            # Extract content based on response type
+            if hasattr(llm_response, 'content'):
+                generated_answer = llm_response.content
+            elif isinstance(llm_response, str):
+                generated_answer = llm_response.replace(llm_prompt, "").strip() if llm_prompt in llm_response else llm_response
+            else:
+                generated_answer = str(llm_response)
+            
+            # Validate and clean response
+            generated_answer = generated_answer.strip()
+            if not generated_answer or len(generated_answer) < 10:
+                generated_answer = "I'd be happy to help with recipes! Ask me about specific ingredients or dishes."
+            
+            # Limit response length
+            if len(generated_answer) > 300:
+                answer_sentences = generated_answer.split('. ')
+                generated_answer = '. '.join(answer_sentences[:2]) + '.' if len(answer_sentences) > 1 else generated_answer[:300]
+            
+            logger.info(f"✅ Response generated ({len(generated_answer)} chars)")
+            return generated_answer
+            
+        except Exception as error:
+            logger.error(f"❌ Simple chat completion error: {str(error)}")
+            return f"Sorry, I encountered an error: {str(error)}"
+
+# Create global LLM service instance
+llm_service = LLMService()

backend/services/vector_store.py

@@ -0,0 +1,386 @@
+# Vector Store Service - Simple setup for retriever use
+import json
+import os
+import shutil
+from typing import List, Dict, Any, Optional
+from pathlib import Path
+
+# Core LangChain imports (always needed)
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.schema import Document
+
+# Local imports
+from backend.config.settings import settings
+from backend.config.database import db_settings
+from backend.config.logging_config import get_logger
+
+# MongoDB imports
+from pymongo import MongoClient
+from backend.services.custom_mongo_vector import CustomMongoDBVectorStore, VectorSearchOptions
+
+# Setup logging
+logger = get_logger("vector_store")
+
+class VectorStoreService:
+    """Simple vector store service - creates or retrieves vector store for retriever use"""
+    
+    def __init__(self):
+        logger.info("📚 Initializing Vector Store Service...")
+        
+        try:
+            self.embeddings = self._get_embeddings()
+            logger.info("✅ Embeddings setup completed")
+            
+            self.vector_store = self._get_or_create_vector_store()
+            logger.info("✅ Vector store setup completed")
+            
+            logger.info("🚀 Vector Store Service initialization successful")
+            
+        except Exception as e:
+            logger.error(f"❌ Vector Store Service initialization failed: {str(e)}", exc_info=True)
+            raise
+    
+    def _get_embeddings(self):
+        """Get embeddings provider based on configuration with conditional imports"""
+        embedding_config = settings.get_embedding_config()
+        provider = embedding_config["provider"]
+        
+        logger.info(f"🔧 Setting up embeddings provider: {provider}")
+        
+        if provider == "openai":
+            try:
+                from langchain_openai import OpenAIEmbeddings
+                logger.info("✅ OpenAI embeddings imported successfully")
+                return OpenAIEmbeddings(
+                    openai_api_key=embedding_config["api_key"],
+                    model=embedding_config["model"]
+                )
+            except ImportError as e:
+                logger.error(f"❌ OpenAI embeddings not available: {e}")
+                raise ImportError("OpenAI provider selected but langchain_openai not installed")
+        
+        elif provider == "google":
+            try:
+                from langchain_google_genai import GoogleGenerativeAIEmbeddings
+                logger.info("✅ Google embeddings imported successfully")
+                return GoogleGenerativeAIEmbeddings(
+                    google_api_key=embedding_config["api_key"],
+                    model=embedding_config["model"]
+                )
+            except ImportError as e:
+                logger.error(f"❌ Google embeddings not available: {e}")
+                raise ImportError("Google provider selected but langchain_google_genai not installed")
+        
+        elif provider == "huggingface":
+            try:
+                # Try modern langchain-huggingface first
+                from langchain_huggingface import HuggingFaceEmbeddings
+                logger.info("✅ HuggingFace embeddings imported successfully")
+                return HuggingFaceEmbeddings(
+                    model_name=embedding_config["model"]
+                )
+            except ImportError:
+                try:
+                    # Fallback to sentence-transformers directly
+                    from sentence_transformers import SentenceTransformer
+                    logger.warning("⚠️ Using sentence-transformers directly (langchain-huggingface not available)")
+                    # Return a wrapper that mimics the embeddings interface
+                    return self._create_sentence_transformer_wrapper(embedding_config["model"])
+                except ImportError as e:
+                    logger.error(f"❌ HuggingFace embeddings not available: {e}")
+                    logger.error("💡 To fix this, install sentence-transformers: pip install sentence-transformers")
+                    raise ImportError("HuggingFace provider selected but sentence-transformers not installed. Run: pip install sentence-transformers")
+        
+        elif provider == "ollama":
+            try:
+                from langchain_community.embeddings import OllamaEmbeddings
+                logger.info("✅ Ollama embeddings imported successfully")
+                return OllamaEmbeddings(
+                    base_url=embedding_config["base_url"],
+                    model=embedding_config["model"]
+                )
+            except ImportError as e:
+                logger.error(f"❌ Ollama embeddings not available: {e}")
+                raise ImportError("Ollama provider selected but langchain_community not installed")
+        
+        else:
+            logger.warning(f"⚠️ Unknown embedding provider '{provider}', falling back to OpenAI")
+            try:
+                from langchain_openai import OpenAIEmbeddings
+                return OpenAIEmbeddings()
+            except ImportError:
+                logger.error("❌ No valid embedding provider available")
+                raise ImportError("No valid embedding provider available")
+    
+    def _create_sentence_transformer_wrapper(self, model_name):
+        """Create a simple wrapper for sentence-transformers to work with LangChain"""
+        from sentence_transformers import SentenceTransformer
+        
+        class SentenceTransformerWrapper:
+            def __init__(self, model_name):
+                self.model = SentenceTransformer(model_name)
+            
+            def encode(self, texts):
+                return self.model.encode(texts).tolist()
+            
+            def embed_query(self, text):
+                return self.model.encode([text])[0].tolist()
+        
+        return SentenceTransformerWrapper(model_name)
+    
+    def _get_or_create_vector_store(self):
+        """Get or create vector store with conditional imports"""
+        db_config = db_settings.get_vector_store_config()
+        provider = db_config["provider"]
+        
+        if provider == "chromadb":
+            try:
+                from langchain_chroma import Chroma
+                
+                persist_dir = Path(db_config["persist_directory"])
+                collection_name = db_config["collection_name"]
+                refresh_on_start = db_config.get("refresh_on_start", False)
+                
+                # Check if refresh is requested
+                if refresh_on_start and persist_dir.exists():
+                    logger.info(f"🔄 CHROMADB_REFRESH_ON_START=true - Deleting existing ChromaDB at {persist_dir}")
+                    shutil.rmtree(persist_dir)
+                    logger.info(f"✅ Existing ChromaDB deleted successfully")
+                
+                # Check if persisted database exists
+                if persist_dir.exists() and any(persist_dir.iterdir()):
+                    logger.info(f"📂 Loading existing ChromaDB from {persist_dir}")
+                    return Chroma(
+                        collection_name=collection_name,
+                        embedding_function=self.embeddings,
+                        persist_directory=str(persist_dir)
+                    )
+                else:
+                    # Create new vector store with documents
+                    logger.info(f"🆕 Creating new ChromaDB at {persist_dir}")
+                    documents = self._load_documents_from_folder()
+                    
+                    if documents:
+                        vector_store = Chroma.from_documents(
+                            documents=documents,
+                            embedding=self.embeddings,
+                            collection_name=collection_name,
+                            persist_directory=str(persist_dir)
+                        )
+                        logger.info(f"✅ Created ChromaDB with {len(documents)} document chunks")
+                        return vector_store
+                    else:
+                        logger.info("📝 No documents found, creating empty ChromaDB")
+                        return Chroma(
+                            collection_name=collection_name,
+                            embedding_function=self.embeddings,
+                            persist_directory=str(persist_dir)
+                        )
+            except ImportError as e:
+                logger.error(f"❌ ChromaDB not available: {e}")
+                raise ImportError("ChromaDB provider selected but langchain_chroma not installed")
+        
+        elif provider == "mongodb":
+            try: 
+                logger.info("🔗 Setting up MongoDB Atlas connection...")
+                client = MongoClient(db_config["uri"])
+                client.admin.command('ping')
+                logger.info(f"✅ MongoDB Atlas connection verified")
+                print(client.list_database_names())
+                # Get the collection
+                database = client[db_config["database"]]
+                collection = database[db_config["collection_name"]]
+                # Create streamlined vector store with Atlas Vector Search
+                options = VectorSearchOptions(
+                    index_name=db_config.get("index_name", "vector_index"),
+                    embedding_key=db_config.get("vector_field", "ingredients_emb"),
+                    text_key="title",
+                    num_candidates=db_config.get("num_candidates", 50),
+                    similarity_metric=db_config.get("similarity_metric", "cosine")
+                )
+                
+                vector_store = CustomMongoDBVectorStore(
+                    collection=collection,
+                    embedding_function=self.embeddings,
+                    options=options
+                )
+                
+                logger.info(f"✅ Custom MongoDB Vector Store created successfully")
+                logger.info("🎯 Using pre-existing embeddings without requiring vector search index")
+                return vector_store
+                
+            except ImportError as e:
+                logger.error(f"❌ MongoDB packages not available: {e}")
+                raise ImportError("MongoDB provider selected but langchain-mongodb not installed. Run: pip install langchain-mongodb pymongo")
+            except Exception as e:
+                logger.error(f"❌ MongoDB Atlas connection failed: {e}")
+                raise ConnectionError(f"Failed to connect to MongoDB Atlas: {e}")
+        
+        else:
+            logger.warning(f"⚠️ Unknown vector store provider '{provider}', falling back to ChromaDB")
+            try:
+                from langchain_chroma import Chroma
+                return Chroma(
+                    collection_name="fallback_collection",
+                    embedding_function=self.embeddings,
+                    persist_directory="./vector_store/fallback_chroma"
+                )
+            except ImportError:
+                logger.error("❌ No valid vector store provider available")
+                raise ImportError("No valid vector store provider available")
+
+    def _load_documents_from_folder(self, folder_path: str = "./data/recipes") -> List[Document]:
+        """Load and chunk all documents from folder with UTF-8 encoding, fallback to sample data"""
+        logger.info(f"📄 Loading documents from: {folder_path}")
+        
+        documents = []
+        folder = Path(folder_path)
+        
+        # Check if folder exists and has files
+        has_recipe_files = False
+        if folder.exists():
+            # Check if there are any files in the recipes folder
+            recipe_files = list(folder.rglob("*"))
+            has_recipe_files = any(f.is_file() and f.stat().st_size > 0 for f in recipe_files)
+        
+        # If no recipe files found, use sample data
+        if not has_recipe_files:
+            logger.info(f"📭 No recipe files found in {folder_path}, using sample data")
+            folder_path = "./data"  # Use data folder where sample_recipes.json is located
+            folder = Path(folder_path)
+        
+        if not folder.exists():
+            logger.error(f"❌ Folder does not exist: {folder.absolute()}")
+            return documents
+        
+        # Text splitter for chunking
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=1000,
+            chunk_overlap=200
+        )
+        
+        # Process all text-based files uniformly
+        for file_path in folder.rglob("*"):
+            if file_path.is_file():
+                try:
+                    # Read file content with UTF-8 encoding
+                    with open(file_path, 'r', encoding='utf-8') as f:
+                        content = f.read()
+                    
+                    # Skip empty files
+                    if not content.strip():
+                        continue
+                    
+                    # Handle JSON files specially to format them properly
+                    if file_path.suffix.lower() == '.json':
+                        formatted_content = self._format_json_recipes(content, file_path)
+                        if formatted_content:
+                            content = formatted_content
+                    
+                    # Split content into chunks using text splitter
+                    chunks = text_splitter.split_text(content)
+                    
+                    # Create documents for each chunk
+                    for i, chunk in enumerate(chunks):
+                        documents.append(Document(
+                            page_content=chunk,
+                            metadata={
+                                "source": str(file_path),
+                                "filename": file_path.name,
+                                "chunk_index": i,
+                                "file_type": file_path.suffix
+                            }
+                        ))
+                        
+                except Exception as e:
+                    logger.error(f"❌ Error loading {file_path}: {e}")
+                    continue
+        
+        logger.info(f"✅ Loaded and chunked {len(documents)} document segments")
+        return documents
+
+    def _format_json_recipes(self, json_content: str, file_path: Path) -> str:
+        """Format JSON recipe data into readable text format similar to MongoDB output"""
+        try:
+            import json
+            recipes = json.loads(json_content)
+            
+            # Handle both single recipe object and array of recipes
+            if isinstance(recipes, dict):
+                recipes = [recipes]
+            elif not isinstance(recipes, list):
+                logger.warning(f"⚠️ Unexpected JSON structure in {file_path}")
+                return None
+            
+            formatted_recipes = []
+            
+            for recipe in recipes:
+                if not isinstance(recipe, dict):
+                    continue
+                    
+                # Extract recipe components
+                title = recipe.get("title", "Untitled Recipe")
+                ingredients = recipe.get("ingredients", [])
+                instructions = recipe.get("instructions", "")
+                
+                # Format similar to MongoDB output
+                formatted_content = f"Recipe: {title}\n"
+                
+                if ingredients:
+                    if isinstance(ingredients, list):
+                        formatted_content += f"Ingredients: {', '.join(ingredients)}\n"
+                    else:
+                        formatted_content += f"Ingredients: {ingredients}\n"
+                
+                if instructions:
+                    # Handle both string and list instructions
+                    if isinstance(instructions, list):
+                        formatted_content += f"Instructions: {' '.join(instructions)}"
+                    else:
+                        formatted_content += f"Instructions: {instructions}"
+                
+                # Add metadata if available
+                metadata = recipe.get("metadata", {})
+                if metadata:
+                    formatted_content += f"\n"
+                    for key, value in metadata.items():
+                        if key in ["cook_time", "difficulty", "servings", "category"]:
+                            formatted_content += f"{key.replace('_', ' ').title()}: {value}\n"
+                
+                formatted_recipes.append(formatted_content)
+            
+            # Join all recipes with double newlines
+            result = "\n\n".join(formatted_recipes)
+            logger.info(f"✅ Formatted {len(recipes)} JSON recipes from {file_path.name}")
+            return result
+            
+        except json.JSONDecodeError as e:
+            logger.error(f"❌ Invalid JSON in {file_path}: {e}")
+            return None
+        except Exception as e:
+            logger.error(f"❌ Error formatting JSON recipes from {file_path}: {e}")
+            return None
+
+    def get_retriever(self):
+        """Get retriever for use with ConversationalRetrievalChain"""
+        logger.info("🔍 Creating retriever from vector store...")
+        
+        # For both ChromaDB and MongoDB Atlas, create standard retriever
+        retriever = self.vector_store.as_retriever()
+        
+        # Configure search parameters based on provider
+        if hasattr(self.vector_store, '__class__'):
+            class_name = self.vector_store.__class__.__name__
+            if 'MongoDB' in class_name:
+                # MongoDB Atlas configuration
+                retriever.search_kwargs = {"k": 5}
+                logger.info("🔍 MongoDB Atlas retriever configured with k=5")
+            else:
+                # ChromaDB configuration
+                retriever.search_kwargs = {"k": 5}
+                logger.info("🔍 ChromaDB retriever configured with k=5")
+
+        return retriever
+
+# Create global vector store service instance
+vector_store_service = VectorStoreService()

backend/tests/test_db_settings.py

@@ -0,0 +1,53 @@
+import unittest
+import os
+from unittest.mock import patch
+from backend.config.database import db_settings
+
+class TestMongoAtlasSettings(unittest.TestCase):
+
+    @patch.dict(os.environ, {
+        'VECTOR_STORE_PROVIDER': 'mongodb'
+    })
+    def test_mongo_settings(self):
+        """
+        Test mongodb config contains expected fields when VECTOR_STORE_PROVIDER is set
+        """
+        # reinitialize db_settings instance
+        db_settings.__init__()
+
+        db_config = db_settings.get_vector_store_config()
+        self.assertTrue('collection_name' in db_config)
+        self.assertTrue('index_name' in db_config)
+        self.assertTrue('text_field' in db_config)
+
+class TestChromaDBSettings(unittest.TestCase):
+
+    @patch.dict(os.environ, {
+        'VECTOR_STORE_PROVIDER': 'chromadb'
+    })
+    def test_mongo_settings(self):
+        """
+        Test chromadb config contains expected fields when VECTOR_STORE_PROVIDER is set
+        """
+        # reinitialize db_settings instance
+        db_settings.__init__()
+
+        db_config = db_settings.get_vector_store_config()
+        self.assertTrue('collection_name' in db_config)
+        self.assertTrue('persist_directory' in db_config)
+        self.assertTrue('refresh_on_start' in db_config)
+
+class TestInvalidDBSettings(unittest.TestCase):
+
+    @patch.dict(os.environ, {
+        'VECTOR_STORE_PROVIDER': 'postgres'
+    })
+    def test_mongo_settings(self):
+        """
+        Test invalid db config raises correct error
+        """
+        # reinitialize db_settings instance
+        db_settings.__init__()
+
+        with self.assertRaisesRegex(ValueError, "Unsupported"):
+            db_settings.get_vector_store_config()

backend/tests/test_llm_provider_settings.py

@@ -0,0 +1,39 @@
+import unittest
+import os
+from unittest.mock import patch
+from backend.config.settings import settings
+
+class TestValidLLMProviderSettings(unittest.TestCase):
+
+    @patch.dict(os.environ, {
+        'GOOGLE_API_KEY': 'ivq',
+        'GOOGLE_MAX_TOKENS': '1200',
+        'OPENAI_API_KEY': 'xyz',
+        'OPENAI_MAX_TOKENS': '3800',
+        'LLM_PROVIDER': 'openai'
+    })
+    def test_valid_llm_provider_settings(self):
+        """
+        Test settings object provides correct config when LLM_PROVIDER is set
+        """
+        # reinitialize settings instance
+        settings.__init__()
+
+        llm_config = settings.get_llm_config()
+        self.assertEqual(llm_config['api_key'], 'xyz')
+        self.assertEqual(llm_config['max_tokens'], 3800)
+
+class TestInvalidLLMProviderSettings(unittest.TestCase):
+
+    @patch.dict(os.environ, {
+        'LLM_PROVIDER': 'microsoft'
+    })
+    def test_invalid_llm_provider_settings(self):
+        """
+        Test that improper provider config raises the right error
+        """
+        # reinitialize settings instance
+        settings.__init__()
+
+        with self.assertRaisesRegex(ValueError, "Unsupported"):
+            settings.get_llm_config()

backend/tests/test_llm_service.py

@@ -0,0 +1,26 @@
+import unittest
+import os
+from unittest.mock import patch
+from backend.services.llm_service import llm_service
+from backend.config.settings import settings
+from backend.config.database import db_settings
+
+class TestLLMService(unittest.TestCase):
+
+    @patch.dict(os.environ, {
+        'VECTOR_STORE_PROVIDER': 'chromadb',
+        'LLM_PROVIDER': 'google'
+,    })
+    def test_chat_completion(self):
+        """
+        Test chat completions work, assuming proper config and API keys
+        """
+        # reinitialize globals instance
+        settings.__init__()
+        db_settings.__init__()
+        llm_service.__init__()
+
+        response = llm_service.simple_chat_completion("Hello there 👋🏽")
+        print(response)
+        # There should be a better way to test this
+        self.assertTrue(len(response) > 0)

backend/utils/__init__.py

@@ -0,0 +1,10 @@
+# Utils package - Utility functions and helpers
+from .sanitization import (
+    DataSanitizer,
+    sanitize_user_input
+)
+
+__all__ = [
+    'DataSanitizer',
+    'sanitize_user_input'
+]

backend/utils/helpers.py

@@ -0,0 +1,2 @@
+# Utility functions for the Recipe Recommendation Bot
+

backend/utils/request_dto/chat_response.py

@@ -0,0 +1,4 @@
+from pydantic import BaseModel, Field
+
+class ChatResponse(BaseModel):
+    response: str = Field(..., description="Bot response to the user message")

backend/utils/request_dto/scrape_request.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+class ScrapeRequest(BaseModel):
+    site: str
+    limit: int = 50
+    output_type: str = "json"  # or "mongo"