initial deploy

.gitignore

@@ -0,0 +1,158 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+PIPFILE.lock
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+Pipfile.lock
+
+# poetry
+poetry.lock
+
+# pdm
+.pdm.toml
+
+# PEP 582
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# OS-specific
+Thumbs.db
+Desktop.ini
+
+# Project-specific
+data/
+*.db
+*.sqlite
+logs/
+*.log
+temp/
+tmp/
+

ARCHITECTURE.md

@@ -0,0 +1,555 @@
+# HR Intervals AI Assistant - Architecture Documentation
+
+## Project Overview
+
+An AI-powered bilingual chatbot for nonprofit organizations providing HR support, policy generation, and compliance checking.
+
+**Tech Stack:**
+- Backend: Python 3.12 + LangChain
+- Vector Database: Qdrant Cloud
+- AI Models: OpenAI (GPT-4o-mini, text-embedding-3-small)
+- UI Framework: Gradio
+- Web Scraping: Firecrawl
+- Monitoring: LangSmith (optional)
+- Deployment: Hugging Face Spaces
+
+---
+
+## System Architecture
+
+### High-Level Architecture
+```
+┌─────────────────────────────────────────────────────────────┐
+│                        USER LAYER                            │
+├──────────────────────────┬──────────────────────────────────┤
+│   app.py                 │   admin.py                        │
+│   (Chat Interface)       │   (Admin Interface)               │
+│   - User Q&A             │   - Upload documents              │
+│   - Policy generation    │   - Scrape web pages              │
+│   - View sources         │   - Manage content                │
+└──────────────────────────┴──────────────────────────────────┘
+                    │                    │
+                    ▼                    ▼
+┌─────────────────────────────────────────────────────────────┐
+│                    APPLICATION LAYER                         │
+├──────────────┬─────────────────┬────────────────────────────┤
+│ chatbot.py   │ ingestion.py    │ scraper.py                 │
+│ - RAG chain  │ - PDF/DOCX      │ - Web scraping             │
+│ - Retrieval  │ - Text chunking │ - URL processing           │
+│ - QA logic   │ - Metadata      │ - Content storage          │
+└──────────────┴─────────────────┴────────────────────────────┘
+                    │                    │
+                    ▼                    ▼
+┌─────────────────────────────────────────────────────────────┐
+│                   EXTERNAL SERVICES                          │
+├─────────────┬─────────────┬───────────────┬─────────────────┤
+│ Qdrant      │ OpenAI      │ Firecrawl     │ LangSmith       │
+│ Cloud       │ API         │ API           │ (optional)      │
+│ - Vectors   │ - Embeddings│ - Scraping    │ - Monitoring    │
+│ - Search    │ - Chat      │ - Markdown    │ - Debugging     │
+└─────────────┴─────────────┴───────────────┴─────────────────┘
+```
+
+---
+
+## Module Relationships
+
+### Core Modules
+
+#### 1. `src/ingestion.py` - Document Processing Module
+
+**Purpose:** Load, process, and store PDF/DOCX documents into vector database
+
+**Key Functions:**
+```python
+create_vectorstore() -> (vectorstore, embeddings, client)
+load_document(file_path: str) -> List[Document]
+chunk_documents(documents, chunk_size=1000, chunk_overlap=200) -> List[Document]
+add_metadata(chunks, source_name, doc_type="document") -> List[Document]
+ingest_document(file_path: str, doc_type="document") -> int
+```
+
+**Dependencies:**
+- `langchain_community.document_loaders` (PyPDFLoader, Docx2txtLoader)
+- `langchain.text_splitter` (RecursiveCharacterTextSplitter)
+- `langchain_openai` (OpenAIEmbeddings)
+- `langchain_qdrant` (QdrantVectorStore)
+- `qdrant_client` (QdrantClient)
+
+**Used By:**
+- `admin.py` (upload functionality)
+
+---
+
+#### 2. `src/scraper.py` - Web Scraping Module
+
+**Purpose:** Scrape web pages and store content in vector database
+
+**Key Functions:**
+```python
+scrape_url(url: str) -> str
+process_and_store_webpage(url: str) -> int
+```
+
+**Dependencies:**
+- `firecrawl` (FirecrawlApp)
+- `langchain.schema` (Document)
+- `langchain.text_splitter` (RecursiveCharacterTextSplitter)
+- `langchain_openai` (OpenAIEmbeddings)
+- `langchain_qdrant` (QdrantVectorStore)
+
+**Used By:**
+- `admin.py` (URL scraping functionality)
+
+---
+
+#### 3. `src/chatbot.py` - RAG Question-Answering Module
+
+**Purpose:** Handle user questions using Retrieval-Augmented Generation
+
+**Key Functions:**
+```python
+create_rag_chain() -> ConversationalRetrievalChain
+ask_question(qa_chain, question: str) -> (answer: str, sources: List[Document])
+```
+
+**Components:**
+- Vector store retriever (k=5 similar documents)
+- LLM: GPT-4o-mini (temperature=0.3)
+- Conversation memory (ConversationBufferMemory)
+- System prompt with disclaimers
+
+**Dependencies:**
+- `langchain_openai` (ChatOpenAI, OpenAIEmbeddings)
+- `langchain_qdrant` (QdrantVectorStore)
+- `langchain.chains` (ConversationalRetrievalChain)
+- `langchain.memory` (ConversationBufferMemory)
+- `qdrant_client` (QdrantClient)
+
+**Used By:**
+- `app.py` (chat interface)
+
+---
+
+### User Interface Modules
+
+#### 4. `app.py` - Chat Interface (End Users)
+
+**Purpose:** Gradio-based chat interface for nonprofit users
+
+**Features:**
+- Real-time Q&A
+- PII detection and warnings
+- Source citations
+- Disclaimer display
+- Conversation history
+- Example questions
+
+**Calls:**
+- `src/chatbot.py` → `create_rag_chain()`, `ask_question()`
+
+**Port:** 7860
+
+---
+
+#### 5. `admin.py` - Admin Interface (Content Managers)
+
+**Purpose:** Gradio-based management interface for HR Intervals team
+
+**Features:**
+- View all documents
+- Upload PDF/DOCX files
+- Scrape single/multiple URLs
+- Delete documents by source
+- Update/replace documents
+
+**Calls:**
+- `src/ingestion.py` → `ingest_document()`
+- `src/scraper.py` → `process_and_store_webpage()`
+- `qdrant_client.QdrantClient` → direct CRUD operations
+
+**Port:** 7861
+
+---
+
+## Data Flow Diagrams
+
+### Flow 1: Document Upload
+```
+User (admin.py)
+    ↓
+    [Select PDF/DOCX file]
+    ↓
+admin.py: upload_document()
+    ↓
+ingestion.py: ingest_document()
+    ↓
+    [Load document] → PyPDFLoader / Docx2txtLoader
+    ↓
+    [Split into chunks] → RecursiveCharacterTextSplitter
+    │   - chunk_size: 1000
+    │   - chunk_overlap: 200
+    ↓
+    [Add metadata]
+    │   - source: filename
+    │   - type: document/policy/guide
+    │   - upload_date: YYYY-MM-DD
+    ↓
+    [Generate embeddings] → OpenAI text-embedding-3-small
+    ↓
+    [Store vectors + metadata] → Qdrant Cloud
+    ↓
+✅ Success: N chunks uploaded
+```
+
+---
+
+### Flow 2: Web Scraping
+```
+User (admin.py)
+    ↓
+    [Enter URL(s)]
+    ↓
+admin.py: scrape_single_url() / scrape_multiple_urls()
+    ↓
+scraper.py: process_and_store_webpage()
+    ↓
+    [Scrape webpage] → Firecrawl API
+    │   - Returns: Markdown content
+    ↓
+    [Create document with metadata]
+    │   - source: URL
+    │   - type: webpage
+    │   - upload_date: YYYY-MM-DD
+    ↓
+    [Split into chunks] → RecursiveCharacterTextSplitter
+    ↓
+    [Generate embeddings] → OpenAI text-embedding-3-small
+    ↓
+    [Store vectors + metadata] → Qdrant Cloud
+    ↓
+✅ Success: N chunks uploaded
+```
+
+---
+
+### Flow 3: Question Answering (RAG)
+```
+User (app.py)
+    ↓
+    [Type question]
+    ↓
+app.py: chat()
+    ↓
+    [Check for PII] → Regex patterns
+    │   - Capitalized names: [A-Z][a-z]+ [A-Z][a-z]+
+    │   - If detected: Show warning
+    ↓
+chatbot.py: ask_question()
+    ↓
+ConversationalRetrievalChain
+    ↓
+    [Convert question to embedding] → OpenAI text-embedding-3-small
+    ↓
+    [Similarity search] → Qdrant Cloud
+    │   - Retrieve top 5 similar chunks
+    │   - Return: chunks + metadata
+    ↓
+    [Combine context + question + chat history]
+    ↓
+    [Generate answer] → OpenAI GPT-4o-mini
+    │   - Temperature: 0.3
+    │   - System prompt: HR assistant with disclaimers
+    ↓
+    [Return answer + source documents]
+    ↓
+app.py: Display answer with sources
+    ↓
+User sees:
+    - Answer
+    - ⚠️ PII warning (if applicable)
+    - 📚 Sources (top 3)
+```
+
+---
+
+### Flow 4: Document Deletion
+```
+User (admin.py)
+    ↓
+    [Enter document name or URL]
+    ↓
+admin.py: delete_document()
+    ↓
+Qdrant Client: delete()
+    ↓
+    [Filter by metadata]
+    │   - Field: "source"
+    │   - Match: exact document name
+    ↓
+    [Delete all matching points]
+    ↓
+✅ Success: All chunks from source deleted
+```
+
+---
+
+### Flow 5: Document Update
+```
+User (admin.py)
+    ↓
+    [Specify old document name]
+    [Select new file]
+    ↓
+admin.py: update_document()
+    ↓
+    [Step 1: Delete old document]
+    │   └─→ delete_document(old_source)
+    ↓
+    [Step 2: Upload new document]
+    │   └─→ upload_document(new_file)
+    ↓
+✅ Success: Document replaced
+```
+
+---
+
+## Configuration
+
+### Environment Variables (`.env`)
+```bash
+# OpenAI API
+OPENAI_API_KEY=sk-proj-...
+OPEN_AI_EMBEDDING_MODEL=text-embedding-3-small
+OPEN_AI_CHAT_MODEL=gpt-4o-mini
+
+# Qdrant Cloud
+QDRANT_URL=https://xxx.cloud.qdrant.io:6333
+QDRANT_API_KEY=xxx
+QDRANT_COLLECTION=hr-intervals
+
+# Firecrawl
+FIRECRAWL_API_KEY=fc-xxx
+
+# LangSmith (Optional)
+LANGSMITH_TRACING=false
+LANGSMITH_API_KEY=xxx
+LANGSMITH_PROJECT=hr-intervals-chatbot
+```
+
+---
+
+## Project Structure
+```
+hr-intervals-chatbot/
+├── src/
+│   ├── __init__.py
+│   ├── ingestion.py          # Document processing
+│   ├── chatbot.py             # RAG Q&A logic
+│   └── scraper.py             # Web scraping
+├── data/
+│   ├── documents/             # Uploaded files
+│   └── scraped/               # Scraped content (cache)
+├── app.py                     # User chat interface
+├── admin.py                   # Admin management interface
+├── .env                       # API keys and config
+├── requirements.txt           # Python dependencies
+├── ARCHITECTURE.md            # This file
+└── README.md                  # Project overview
+```
+
+---
+
+## Key Technical Decisions
+
+### 1. Vector Database: Qdrant Cloud
+- **Why:** Built-in web UI, easy document management, free tier
+- **Alternative considered:** Pinecone (limited free tier, no document-level UI)
+
+### 2. Embedding Model: text-embedding-3-small
+- **Dimensions:** 1536
+- **Why:** Excellent quality with best cost-performance ratio, multilingual support (English/French)
+
+### 3. LLM: GPT-4o-mini
+- **Why:** Cost-effective, sufficient for HR Q&A, fast response
+- **Alternative:** GPT-4o (more expensive but higher quality)
+
+### 4. Chunking Strategy
+- **Chunk size:** 1000 characters
+- **Overlap:** 200 characters
+- **Separators:** `["\n\n", "\n", ". ", " ", ""]`
+- **Why:** Balances context preservation and retrieval accuracy
+
+### 5. Retrieval: Top-k similarity search
+- **k=5:** Retrieve 5 most similar chunks
+- **Distance metric:** Cosine similarity
+- **Why:** Good balance between context and noise
+
+---
+
+## Metadata Schema
+
+Every chunk stored in Qdrant has the following metadata:
+```python
+{
+    "source": str,           # Filename or URL
+    "type": str,             # "document" | "webpage" | "policy" | "guide"
+    "upload_date": str,      # "YYYY-MM-DD"
+    "page": int,             # (optional) Page number for PDFs
+    "valid_until": str,      # (optional) Expiry date for policies
+    "version": str,          # (optional) Version number
+}
+```
+
+---
+
+## Document Management Operations
+
+### View Documents
+```python
+# List all unique documents
+client.scroll(collection_name, limit=1000, with_payload=True)
+# Group by 'source' field
+```
+
+### Upload Document
+```python
+# 1. Load: PyPDFLoader / Docx2txtLoader
+# 2. Chunk: RecursiveCharacterTextSplitter
+# 3. Add metadata: source, type, date
+# 4. Embed: OpenAI text-embedding-3-small
+# 5. Store: QdrantVectorStore.from_documents()
+```
+
+### Delete Document
+```python
+client.delete(
+    collection_name=collection_name,
+    points_selector=FilterSelector(
+        filter=Filter(
+            must=[
+                FieldCondition(
+                    key="source",
+                    match=MatchValue(value="filename.pdf")
+                )
+            ]
+        )
+    )
+)
+```
+
+### Update Document
+```python
+# 1. Delete old version (by source name)
+# 2. Upload new version
+```
+
+---
+
+## Security Features
+
+### PII Detection
+- Regex pattern for names: `\b[A-Z][a-z]+ [A-Z][a-z]+\b`
+- Warning displayed to user if detected
+- Future: Integrate Microsoft Presidio for advanced PII detection
+
+### Disclaimers
+- Shown on first interaction
+- Embedded in system prompt
+- Reminds users to consult professionals
+
+### API Key Security
+- Stored in `.env` file (not in version control)
+- `.env` added to `.gitignore`
+
+---
+
+## Performance Considerations
+
+### Embedding Cost
+- Model: text-embedding-3-small
+- Cost: ~$0.13 per 1M tokens
+- Typical document: 10 pages ≈ 5,000 tokens ≈ $0.0007
+
+### Chat Cost
+- Model: GPT-4o-mini
+- Cost: ~$0.15 per 1M input tokens, $0.60 per 1M output tokens
+- Typical query: 5 chunks (5,000 tokens) + question (100 tokens) ≈ $0.0008
+
+### Storage
+- Qdrant free tier: 1 GB
+- Each chunk: ~1 KB metadata + 12 KB vector (3072 dims × 4 bytes)
+- Capacity: ~75,000 chunks (approximately 1,500 documents of 50 chunks each)
+
+---
+
+## Future Enhancements
+
+### Phase 1 (Week 9-12) - Policy Features
+- Policy template library
+- Policy generation from user input
+- Policy compliance checking
+- Risk identification
+
+### Phase 2 (Week 13-18) - Advanced Features
+- Bilingual support (French)
+- Language detection and switching
+- Content recommendation system
+- Feedback collection mechanism
+
+### Phase 3 (Week 19-20) - Production
+- Deployment to Hugging Face Spaces
+- User authentication (if needed)
+- Analytics dashboard
+- Automated expiry detection for policies
+
+---
+
+## Troubleshooting
+
+### Common Issues
+
+**1. "Collection not found" error**
+```bash
+# Solution: Collection is created automatically on first upload
+# Just upload a document and it will be created
+```
+
+**2. "No documents found" when asking questions**
+```bash
+# Solution: Upload at least one document first via admin.py
+```
+
+**3. "Rate limit exceeded" from OpenAI**
+```bash
+# Solution: Add delays between requests or upgrade OpenAI plan
+```
+
+**4. "Firecrawl scraping failed"**
+```bash
+# Solution: Check if URL is accessible, verify Firecrawl API key
+```
+
+---
+
+## Development Timeline
+
+- **Week 1-2:** Infrastructure setup ✅
+- **Week 3-4:** Basic RAG system ✅
+- **Week 5-6:** Web scraping + chat interface
+- **Week 7-8:** Quality improvements
+- **Week 9-10:** Admin interface
+- **Week 11-12:** Demo delivery
+- **Week 13-16:** Policy features
+- **Week 17-18:** Bilingual support
+- **Week 19-20:** Final delivery
+
+---
+
+## References
+
+- LangChain Documentation: https://python.langchain.com/docs/
+- Qdrant Documentation: https://qdrant.tech/documentation/
+- OpenAI API Reference: https://platform.openai.com/docs/
+- Gradio Documentation: https://www.gradio.app/docs/
+```

README.md

@@ -0,0 +1,27 @@
+---
+title: HR Intervals Chatbot
+emoji: 💼
+colorFrom: blue
+colorTo: purple
+sdk: gradio
+sdk_version: 5.49.0
+app_file: app.py
+pinned: false
+---
+
+# HR Intervals AI Assistant
+
+A RAG-powered chatbot that provides HR knowledge and policy guidance for non-profit organizations.
+
+## Features
+- 🤖 AI-powered Q&A based on HR knowledge base
+- 📚 Source citations for transparency
+- ⚠️ PII detection and warnings
+- 💬 Interactive chat interface
+
+## Setup
+This Space requires the following environment variables to be set:
+- `OPENAI_API_KEY`: Your OpenAI API key
+- `QDRANT_URL`: Your Qdrant vector database URL
+- `QDRANT_API_KEY`: Your Qdrant API key
+

admin.py

@@ -0,0 +1,422 @@
+"""
+Gradio admin interface for content management
+Allows uploading documents, scraping URLs, and managing content
+"""
+
+import gradio as gr
+import os
+from dotenv import load_dotenv
+from qdrant_client import QdrantClient, models
+from src.ingestion import ingest_document
+from src.scraper import process_and_store_webpage
+
+load_dotenv()
+
+# Initialize Qdrant client
+client = QdrantClient(
+    url=os.getenv("QDRANT_URL"),
+    api_key=os.getenv("QDRANT_API_KEY")
+)
+collection_name = os.getenv("QDRANT_COLLECTION")
+
+# Create index for metadata.source to enable filtering
+try:
+    client.create_payload_index(
+        collection_name=collection_name,
+        field_name="metadata.source",
+        field_schema=models.PayloadSchemaType.KEYWORD
+    )
+    print("✅ Payload index for metadata.source created successfully")
+except Exception as e:
+    # Index might already exist or collection not found
+    print(f"ℹ️ Index status: {str(e)}")
+
+
+# ==================== Functions ====================
+
+def list_all_documents():
+    """
+    List all uploaded documents
+    
+    Returns:
+        HTML table string with selectable content
+    """
+    try:
+        # Paginate through ALL points (Qdrant has 5800+ points)
+        all_points = []
+        offset = None
+        
+        while True:
+            result = client.scroll(
+                collection_name=collection_name,
+                limit=1000,
+                offset=offset,
+                with_payload=True
+            )
+            points, next_offset = result
+            all_points.extend(points)
+            
+            if next_offset is None:
+                break
+            offset = next_offset
+        
+        # Group by source
+        docs_dict = {}
+        for point in all_points:
+            payload = point.payload
+            # Metadata is nested inside payload
+            metadata = payload.get("metadata", {})
+            source = metadata.get("source", "Unknown")
+            
+            if source not in docs_dict:
+                docs_dict[source] = {
+                    "name": source,
+                    "type": metadata.get("type", "Unknown"),
+                    "date": metadata.get("upload_date", "Unknown"),
+                    "chunks": 0
+                }
+            docs_dict[source]["chunks"] += 1
+        
+        # Create HTML table with selectable text
+        if not docs_dict or (len(docs_dict) == 1 and "Unknown" in docs_dict):
+            return """
+            <div style="padding: 20px; text-align: center; color: #666;">
+                <p>📂 No documents yet</p>
+            </div>
+            """
+        
+        html = """
+        <style>
+            .docs-table {
+                width: 100%;
+                border-collapse: collapse;
+                font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Arial, sans-serif;
+                user-select: text;
+                -webkit-user-select: text;
+                -moz-user-select: text;
+                -ms-user-select: text;
+            }
+            .docs-table thead {
+                background-color: #f8f9fa;
+            }
+            .docs-table th {
+                padding: 12px;
+                text-align: left;
+                font-weight: 600;
+                border-bottom: 2px solid #dee2e6;
+                user-select: text;
+            }
+            .docs-table td {
+                padding: 12px;
+                border-bottom: 1px solid #dee2e6;
+                user-select: text;
+                cursor: text;
+            }
+            .docs-table tr:hover {
+                background-color: #f8f9fa;
+            }
+            .doc-name {
+                color: #0066cc;
+                word-break: break-all;
+            }
+        </style>
+        <table class="docs-table">
+            <thead>
+                <tr>
+                    <th>Document Name</th>
+                    <th>Type</th>
+                    <th>Upload Date</th>
+                    <th>Chunks</th>
+                </tr>
+            </thead>
+            <tbody>
+        """
+        
+        for doc in docs_dict.values():
+            html += f"""
+                <tr>
+                    <td class="doc-name">{doc['name']}</td>
+                    <td>{doc['type']}</td>
+                    <td>{doc['date']}</td>
+                    <td>{doc['chunks']}</td>
+                </tr>
+            """
+        
+        html += """
+            </tbody>
+        </table>
+        """
+        
+        return html
+    
+    except Exception as e:
+        return f"""
+        <div style="padding: 20px; color: #dc3545;">
+            <p>❌ Error: {str(e)}</p>
+        </div>
+        """
+
+
+def upload_document(file, doc_type="document"):
+    """
+    Upload PDF or DOCX file
+    
+    Args:
+        file: Uploaded file object
+        doc_type: Type of document
+        
+    Returns:
+        Success message
+    """
+    if file is None:
+        return "❌ Please select a file"
+    
+    try:
+        file_path = file.name
+        
+        # Ingest document
+        num_chunks = ingest_document(file_path, doc_type)
+        
+        return f"✅ Success!\n\nFile: {os.path.basename(file_path)}\nChunks created: {num_chunks}\nType: {doc_type}"
+    
+    except Exception as e:
+        return f"❌ Upload failed:\n{str(e)}"
+
+
+def scrape_single_url(url):
+    """
+    Scrape single URL
+    
+    Args:
+        url: URL to scrape
+        
+    Returns:
+        Success message
+    """
+    if not url:
+        return "❌ Please enter a URL"
+    
+    try:
+        num_chunks = process_and_store_webpage(url)
+        return f"✅ Success!\n\nURL: {url}\nChunks created: {num_chunks}"
+    
+    except Exception as e:
+        return f"❌ Scraping failed:\n{str(e)}"
+
+
+def scrape_multiple_urls(urls_text):
+    """
+    Scrape multiple URLs
+    
+    Args:
+        urls_text: URLs separated by newlines
+        
+    Returns:
+        Summary of results
+    """
+    if not urls_text:
+        return "❌ Please enter URLs (one per line)"
+    
+    urls = [url.strip() for url in urls_text.split('\n') if url.strip()]
+    
+    results = []
+    success_count = 0
+    fail_count = 0
+    
+    for url in urls:
+        try:
+            num_chunks = process_and_store_webpage(url)
+            results.append(f"✅ {url}: {num_chunks} chunks")
+            success_count += 1
+        except Exception as e:
+            results.append(f"❌ {url}: {str(e)}")
+            fail_count += 1
+    
+    summary = f"📊 Summary: {success_count} succeeded, {fail_count} failed\n\n"
+    return summary + "\n".join(results)
+
+
+def delete_document(source_name):
+    """
+    Delete document by source name
+    
+    Args:
+        source_name: Name or URL of the source
+        
+    Returns:
+        Success message
+    """
+    if not source_name:
+        return "❌ Please enter document name or URL"
+    
+    try:
+        client.delete(
+            collection_name=collection_name,
+            points_selector=models.FilterSelector(
+                filter=models.Filter(
+                    must=[
+                        models.FieldCondition(
+                            key="metadata.source",
+                            match=models.MatchValue(value=source_name)
+                        )
+                    ]
+                )
+            )
+        )
+        
+        return f"✅ Successfully deleted all content from:\n{source_name}"
+    
+    except Exception as e:
+        return f"❌ Deletion failed:\n{str(e)}"
+
+
+# ==================== Gradio Interface (5.49) ====================
+
+with gr.Blocks(
+    title="HR Intervals - Admin Panel", 
+    theme=gr.themes.Soft()
+) as demo:
+    
+    gr.Markdown("# 📁 HR Intervals - Knowledge Base Management")
+    gr.Markdown("Manage documents and web content for the AI assistant")
+    
+    with gr.Tabs():
+        
+        # Tab 1: View Documents
+        with gr.Tab("📋 View Documents"):
+            gr.Markdown("### Current documents in knowledge base")
+            gr.Markdown("💡 *Tip: You can select and copy any text from the table below*")
+            
+            refresh_btn = gr.Button("🔄 Refresh List", variant="primary")
+            
+            docs_table = gr.HTML(
+                label="Documents"
+            )
+            
+            refresh_btn.click(list_all_documents, outputs=docs_table)
+            demo.load(list_all_documents, outputs=docs_table)
+        
+        # Tab 2: Upload Documents
+        with gr.Tab("⬆️ Upload Documents"):
+            gr.Markdown("### Upload PDF or DOCX files")
+            
+            file_input = gr.File(
+                label="Select File (PDF or DOCX)",
+                file_types=[".pdf", ".docx"]
+            )
+            
+            doc_type_input = gr.Radio(
+                choices=["document", "policy", "guide", "article"],
+                value="document",
+                label="Document Type"
+            )
+            
+            upload_btn = gr.Button("📤 Upload", variant="primary", size="lg")
+            upload_output = gr.Textbox(label="Upload Result", lines=5)
+            
+            upload_btn.click(
+                upload_document,
+                inputs=[file_input, doc_type_input],
+                outputs=upload_output
+            )
+        
+        # Tab 3: Scrape URLs
+        with gr.Tab("🌐 Scrape Web Pages"):
+            gr.Markdown("### Scrape content from URLs")
+            
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("#### Single URL")
+                    url_input = gr.Textbox(
+                        label="Enter URL",
+                        placeholder="https://example.com/article"
+                    )
+                    scrape_btn = gr.Button("🔍 Scrape", variant="primary")
+                    scrape_output = gr.Textbox(label="Result", lines=4)
+                    
+                    scrape_btn.click(
+                        scrape_single_url,
+                        inputs=url_input,
+                        outputs=scrape_output
+                    )
+                
+                with gr.Column():
+                    gr.Markdown("#### Batch URLs")
+                    urls_input = gr.Textbox(
+                        label="Enter multiple URLs (one per line)",
+                        placeholder="https://example.com/page1\nhttps://example.com/page2",
+                        lines=6
+                    )
+                    batch_btn = gr.Button("🔍 Batch Scrape", variant="primary")
+                    batch_output = gr.Textbox(label="Batch Results", lines=8)
+                    
+                    batch_btn.click(
+                        scrape_multiple_urls,
+                        inputs=urls_input,
+                        outputs=batch_output
+                    )
+        
+        # Tab 4: Delete Documents
+        with gr.Tab("🗑️ Delete Documents"):
+            gr.Markdown("### Delete documents or web pages")
+            gr.Markdown("⚠️ **Warning**: This operation cannot be undone!")
+            
+            delete_input = gr.Textbox(
+                label="Document Name or URL",
+                placeholder="e.g., hiring_policy.pdf or https://example.com/article"
+            )
+            
+            delete_btn = gr.Button("🗑️ Delete", variant="stop", size="lg")
+            delete_output = gr.Textbox(label="Delete Result", lines=3)
+            
+            delete_btn.click(
+                delete_document,
+                inputs=delete_input,
+                outputs=delete_output
+            )
+        
+        # Tab 5: Help
+        with gr.Tab("ℹ️ Help"):
+            gr.Markdown("""
+            ### Usage Guide
+            
+            #### 📋 View Documents
+            - Shows all uploaded documents and web pages
+            - Displays document type, upload date, and number of chunks
+            - Click "Refresh" to see the latest status
+            
+            #### ⬆️ Upload Documents
+            - Supports PDF and DOCX formats
+            - Documents are automatically split into chunks (~1000 characters each)
+            - You can categorize documents by type
+            
+            #### 🌐 Scrape Web Pages
+            - Enter full URLs (including https://)
+            - Supports single or batch scraping
+            - Content is automatically converted to Markdown format
+            
+            #### 🗑️ Delete Documents
+            - Enter exact filename or URL
+            - Deletes all chunks from that source
+            - **Warning**: Cannot be undone!
+            - **Tip**: To update a document, delete it first then upload the new version
+            
+            ---
+            
+            ### Advanced Management
+            
+            For detailed vector database management, visit:
+            [Qdrant Cloud Dashboard](https://cloud.qdrant.io)
+            
+            ### Technical Support
+            
+            If you encounter issues, please contact the development team.
+            """)
+
+
+if __name__ == "__main__":
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7861,
+        share=False
+    )

app.py

@@ -0,0 +1,149 @@
+"""
+Gradio chat interface for end users
+Uses Gradio 5.49 ChatInterface API
+"""
+
+import gradio as gr
+import os
+from dotenv import load_dotenv
+from src.chatbot import create_rag_chain, ask_question
+import re
+import uuid
+
+load_dotenv()
+
+# Initialize chatbot
+print("🤖 Initializing chatbot...")
+rag_chain, retriever = create_rag_chain()
+print("✅ Chatbot ready!")
+
+# Generate unique session ID for each user
+session_id = str(uuid.uuid4())
+
+
+def check_pii(text: str) -> bool:
+    """
+    Simple PII detection - checks for potential names
+    
+    Args:
+        text: Input text to check
+        
+    Returns:
+        True if PII detected
+    """
+    # Check for capitalized words that might be names
+    name_pattern = r'\b[A-Z][a-z]+ [A-Z][a-z]+\b'
+    if re.search(name_pattern, text):
+        return True
+    return False
+
+
+def chat_response(message: str, history: list) -> str:
+    """
+    Handle chat messages (Gradio 5.x format)
+    
+    Args:
+        message: User's message
+        history: Conversation history
+        
+    Returns:
+        Bot's response
+    """
+    
+    # Check for PII
+    warning = ""
+    if check_pii(message):
+        warning = "⚠️ **Warning**: Please avoid sharing personal information about specific individuals.\n\n"
+    
+    # Get answer from chatbot
+    try:
+        answer, sources = ask_question(rag_chain, retriever, message, session_id)
+        
+        # Format response with sources
+        response = warning + answer
+        
+        if sources:
+            response += "\n\n📚 **Sources:**\n"
+            for i, doc in enumerate(sources[:3], 1):
+                source = doc.metadata.get("source", "Unknown")
+                response += f"{i}. {source}\n"
+        
+        return response
+    
+    except Exception as e:
+        return f"❌ Error: {str(e)}\n\nPlease make sure documents have been uploaded to the system."
+
+
+# Create Gradio interface (Gradio 5.49 API)
+with gr.Blocks(
+    title="HR Intervals AI Assistant", 
+    theme=gr.themes.Soft()
+) as demo:
+    
+    gr.Markdown("""
+    # 💼 HR Intervals AI Assistant
+    
+    Get instant answers to your HR questions based on our knowledge base.
+    """)
+    
+    # Disclaimer
+    with gr.Accordion("⚠️ Important Disclaimer - Please Read", open=False):
+        gr.Markdown("""
+        **This tool is designed to provide general HR-related information and draft policy suggestions.**
+        
+        - This is **NOT** a substitute for professional legal or HR advice
+        - For legal compliance and important decisions, consult a qualified attorney or HR professional
+        - Do **NOT** share personal information about specific individuals
+        
+        By using this tool, you acknowledge that you understand these limitations.
+        """)
+    
+    # Welcome message with disclaimer and example questions
+    WELCOME_MESSAGE = """👋 **Welcome to the HR Intervals AI Assistant!**
+
+⚠️ **Important Disclaimer:**
+
+This tool is designed to provide general HR-related information and draft policy suggestions. It is not a substitute for professional legal or HR advice. For legal compliance and to ensure the best outcome for your organization, we recommend consulting a qualified attorney or HR professional before implementing any policies or making decisions based on the information provided.
+
+---
+
+How can I help you today? **Try asking:**
+
+• What should I include in a remote work policy?
+• How do I handle employee terminations properly?
+• What are best practices for hiring in Canada?
+• Tell me about workplace safety requirements"""
+    
+    # Chat interface (Gradio 5.x ChatInterface)
+    chat_interface = gr.ChatInterface(
+        fn=chat_response,
+        chatbot=gr.Chatbot(
+            height=500,
+            show_label=False,
+            type='messages',
+            avatar_images=(None, "https://em-content.zobj.net/thumbs/120/apple/354/robot_1f916.png"),
+            value=[{"role": "assistant", "content": WELCOME_MESSAGE}]
+        ),
+        textbox=gr.Textbox(
+            placeholder="Ask your HR question here...",
+            container=False,
+            scale=7
+        ),
+        title="",
+        description="",
+        theme=gr.themes.Soft()
+    )
+    
+    # Footer
+    gr.Markdown("""
+    ---
+    💡 **Tip**: Be specific in your questions for better answers. Remember to consult professionals for legal matters.
+    """)
+
+
+if __name__ == "__main__":
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False
+    )

chatbot-widget.html

@@ -0,0 +1,336 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>HR Chatbot Widget - Bottom Right Corner</title>
+    <style>
+        /* Demo page styles */
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+        }
+
+        body {
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+            background: #f8fafc;
+            min-height: 100vh;
+            padding: 40px;
+        }
+
+        .demo-content {
+            max-width: 800px;
+            margin: 0 auto;
+        }
+
+        .demo-content h1 {
+            font-size: 2rem;
+            color: #1e293b;
+            margin-bottom: 20px;
+        }
+
+        .demo-content p {
+            color: #64748b;
+            line-height: 1.8;
+            margin-bottom: 15px;
+        }
+
+        .demo-content .note {
+            background: #fef3c7;
+            border-left: 4px solid #f59e0b;
+            padding: 16px 20px;
+            border-radius: 0 8px 8px 0;
+            margin: 30px 0;
+        }
+
+        .demo-content .note strong {
+            color: #92400e;
+        }
+
+        /* ============================================
+           CHATBOT WIDGET STYLES - COPY FROM HERE
+           ============================================ */
+        
+        /* Chat Toggle Button */
+        .chat-widget-button {
+            position: fixed;
+            bottom: 24px;
+            right: 24px;
+            width: 64px;
+            height: 64px;
+            border-radius: 50%;
+            background: linear-gradient(135deg, #6366f1 0%, #8b5cf6 100%);
+            border: none;
+            cursor: pointer;
+            box-shadow: 0 8px 32px rgba(99, 102, 241, 0.4);
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
+            z-index: 9998;
+        }
+
+        .chat-widget-button:hover {
+            transform: scale(1.1);
+            box-shadow: 0 12px 40px rgba(99, 102, 241, 0.5);
+        }
+
+        .chat-widget-button svg {
+            width: 28px;
+            height: 28px;
+            fill: white;
+            transition: transform 0.3s ease;
+        }
+
+        .chat-widget-button.active svg {
+            transform: rotate(90deg);
+        }
+
+        /* Notification Badge */
+        .chat-widget-badge {
+            position: absolute;
+            top: -4px;
+            right: -4px;
+            width: 20px;
+            height: 20px;
+            background: #ef4444;
+            border-radius: 50%;
+            color: white;
+            font-size: 12px;
+            font-weight: 600;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            animation: pulse-badge 2s infinite;
+        }
+
+        @keyframes pulse-badge {
+            0%, 100% { transform: scale(1); }
+            50% { transform: scale(1.1); }
+        }
+
+        /* Chat Window */
+        .chat-widget-window {
+            position: fixed;
+            bottom: 100px;
+            right: 24px;
+            width: 400px;
+            height: 600px;
+            background: white;
+            border-radius: 20px;
+            box-shadow: 0 25px 80px rgba(0, 0, 0, 0.2);
+            display: flex;
+            flex-direction: column;
+            overflow: hidden;
+            z-index: 9999;
+            opacity: 0;
+            visibility: hidden;
+            transform: translateY(20px) scale(0.95);
+            transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
+        }
+
+        .chat-widget-window.open {
+            opacity: 1;
+            visibility: visible;
+            transform: translateY(0) scale(1);
+        }
+
+        /* Chat Header */
+        .chat-widget-header {
+            background: linear-gradient(135deg, #6366f1 0%, #8b5cf6 100%);
+            padding: 18px 20px;
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+        }
+
+        .chat-widget-header-info {
+            display: flex;
+            align-items: center;
+            gap: 12px;
+        }
+
+        .chat-widget-avatar {
+            width: 44px;
+            height: 44px;
+            background: rgba(255, 255, 255, 0.2);
+            border-radius: 50%;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            font-size: 24px;
+        }
+
+        .chat-widget-title {
+            color: white;
+        }
+
+        .chat-widget-title h3 {
+            font-size: 1rem;
+            font-weight: 600;
+            margin-bottom: 2px;
+        }
+
+        .chat-widget-title span {
+            font-size: 0.8rem;
+            opacity: 0.9;
+            display: flex;
+            align-items: center;
+            gap: 6px;
+        }
+
+        .chat-widget-title .status-dot {
+            width: 8px;
+            height: 8px;
+            background: #4ade80;
+            border-radius: 50%;
+        }
+
+        .chat-widget-close {
+            background: rgba(255, 255, 255, 0.2);
+            border: none;
+            width: 32px;
+            height: 32px;
+            border-radius: 50%;
+            color: white;
+            cursor: pointer;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            transition: background 0.2s;
+        }
+
+        .chat-widget-close:hover {
+            background: rgba(255, 255, 255, 0.3);
+        }
+
+        /* Chat Body (iframe container) */
+        .chat-widget-body {
+            flex: 1;
+            overflow: hidden;
+        }
+
+        .chat-widget-body iframe {
+            width: 100%;
+            height: 100%;
+            border: none;
+        }
+
+        /* Mobile Responsive */
+        @media (max-width: 480px) {
+            .chat-widget-window {
+                width: calc(100% - 20px);
+                height: calc(100% - 120px);
+                right: 10px;
+                bottom: 90px;
+                border-radius: 16px;
+            }
+
+            .chat-widget-button {
+                width: 56px;
+                height: 56px;
+                bottom: 20px;
+                right: 20px;
+            }
+        }
+
+        /* ============================================
+           END OF CHATBOT WIDGET STYLES
+           ============================================ */
+    </style>
+</head>
+<body>
+    <!-- Demo Page Content -->
+    <div class="demo-content">
+        <h1>Your Website Content Here</h1>
+        <p>This is a demo page showing how the HR Chatbot widget appears in the bottom right corner. The chatbot is always accessible to visitors while they browse your website.</p>
+        <p>Our HR Assistant is powered by advanced AI technology, providing instant answers to your human resources questions. Whether you need guidance on policies, hiring practices, or workplace regulations, our chatbot is here to help.</p>
+        <p>The widget is designed to be non-intrusive while remaining easily accessible. It works seamlessly on both desktop and mobile devices.</p>
+        
+        <div class="note">
+            <strong>👉 Click the chat button in the bottom right corner to open the HR Assistant!</strong>
+        </div>
+        
+        <p>Get instant support for common HR inquiries including employee onboarding, benefits information, leave policies, and workplace compliance. Our AI assistant is available 24/7 to provide helpful guidance.</p>
+        <p>Please note that while our chatbot provides general HR information, it should not replace professional legal or HR advice for important decisions.</p>
+    </div>
+
+    <!-- ============================================
+         CHATBOT WIDGET HTML - COPY FROM HERE
+         ============================================ -->
+    
+    <!-- Chat Toggle Button -->
+    <button class="chat-widget-button" id="chatWidgetButton" onclick="toggleChat()">
+        <span class="chat-widget-badge">1</span>
+        <svg viewBox="0 0 24 24" id="chatIcon">
+            <path d="M20 2H4c-1.1 0-2 .9-2 2v18l4-4h14c1.1 0 2-.9 2-2V4c0-1.1-.9-2-2-2zm0 14H6l-2 2V4h16v12z"/>
+        </svg>
+        <svg viewBox="0 0 24 24" id="closeIcon" style="display: none;">
+            <path d="M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/>
+        </svg>
+    </button>
+
+    <!-- Chat Window -->
+    <div class="chat-widget-window" id="chatWidgetWindow">
+        <div class="chat-widget-header">
+            <div class="chat-widget-header-info">
+                <div class="chat-widget-avatar">🤖</div>
+                <div class="chat-widget-title">
+                    <h3>HR Assistant</h3>
+                    <span><span class="status-dot"></span> Online</span>
+                </div>
+            </div>
+            <button class="chat-widget-close" onclick="toggleChat()">
+                <svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor">
+                    <path d="M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/>
+                </svg>
+            </button>
+        </div>
+        <div class="chat-widget-body">
+            <iframe 
+                src="https://pikamomo-hr-intervals-chatbot.hf.space"
+                title="HR Chatbot"
+                loading="lazy">
+            </iframe>
+        </div>
+    </div>
+
+    <!-- ============================================
+         CHATBOT WIDGET JAVASCRIPT - COPY THIS TOO
+         ============================================ -->
+    <script>
+        let isOpen = false;
+        const button = document.getElementById('chatWidgetButton');
+        const window_el = document.getElementById('chatWidgetWindow');
+        const chatIcon = document.getElementById('chatIcon');
+        const closeIcon = document.getElementById('closeIcon');
+        const badge = document.querySelector('.chat-widget-badge');
+
+        function toggleChat() {
+            isOpen = !isOpen;
+            
+            if (isOpen) {
+                window_el.classList.add('open');
+                button.classList.add('active');
+                chatIcon.style.display = 'none';
+                closeIcon.style.display = 'block';
+                badge.style.display = 'none';
+            } else {
+                window_el.classList.remove('open');
+                button.classList.remove('active');
+                chatIcon.style.display = 'block';
+                closeIcon.style.display = 'none';
+            }
+        }
+
+        // Close on escape key
+        document.addEventListener('keydown', function(e) {
+            if (e.key === 'Escape' && isOpen) {
+                toggleChat();
+            }
+        });
+    </script>
+</body>
+</html>
+

requirements.txt

@@ -0,0 +1,30 @@
+# ======================= LangChain Core =======================
+langchain==1.0.2
+langchain-openai==1.0.1
+langchain-qdrant==1.1.0
+langchain-community==0.4.1
+langchain-core==1.0.1
+
+# ======================= Vector Database =======================
+qdrant-client==1.15.1
+
+# ======================= Document Processing =======================
+pypdf==6.1.1
+python-docx==1.2.0
+unstructured==0.18.15
+
+# ======================= Web Scraping =======================
+firecrawl-py==4.5.0
+
+# ======================= User Interface =======================
+gradio==5.49.1
+
+# ======================= OpenAI =======================
+openai==1.109.1
+
+# ======================= Utilities =======================
+python-dotenv==1.1.1
+requests==2.32.5
+numpy==2.2.6
+pandas==2.2.3
+tiktoken==0.11.0

src/chatbot.py

@@ -0,0 +1,195 @@
+"""
+RAG chatbot module using latest LangChain with LCEL
+Handles question-answering with conversation memory using modern patterns
+"""
+
+import os
+from dotenv import load_dotenv
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from langchain_qdrant import QdrantVectorStore
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_core.chat_history import BaseChatMessageHistory
+from langchain_community.chat_message_histories import ChatMessageHistory
+from langchain_core.runnables import RunnablePassthrough, RunnableLambda
+from langchain_core.runnables.history import RunnableWithMessageHistory
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.documents import Document
+from qdrant_client import QdrantClient
+from typing import Tuple, List, Dict, Any
+from operator import itemgetter
+
+load_dotenv()
+
+# Store for chat sessions
+session_store = {}
+
+
+def get_session_history(session_id: str) -> BaseChatMessageHistory:
+    """
+    Get or create chat history for a session
+    
+    Args:
+        session_id: Unique identifier for the session
+        
+    Returns:
+        Chat message history object
+    """
+    if session_id not in session_store:
+        session_store[session_id] = ChatMessageHistory()
+    return session_store[session_id]
+
+
+def format_docs(docs: List[Document]) -> str:
+    """
+    Format retrieved documents into a single string
+    
+    Args:
+        docs: List of retrieved documents
+        
+    Returns:
+        Formatted string with document contents
+    """
+    return "\n\n".join(doc.page_content for doc in docs)
+
+
+def create_rag_chain():
+    """
+    Create RAG question-answering chain using LCEL (LangChain Expression Language)
+    Modern approach with pipe operator for better composability
+    
+    Returns:
+        Conversational RAG chain with message history
+    """
+    
+    # 1. Connect to Qdrant
+    client = QdrantClient(
+        url=os.getenv("QDRANT_URL"),
+        api_key=os.getenv("QDRANT_API_KEY")
+    )
+    
+    embeddings = OpenAIEmbeddings(
+        model=os.getenv("OPEN_AI_EMBEDDING_MODEL", "text-embedding-3-small")
+    )
+    
+    vectorstore = QdrantVectorStore(
+        client=client,
+        collection_name=os.getenv("QDRANT_COLLECTION"),
+        embedding=embeddings
+    )
+    
+    # 2. Create retriever
+    retriever = vectorstore.as_retriever(
+        search_type="similarity",
+        search_kwargs={"k": 5}
+    )
+    
+    # 3. Create LLM
+    llm = ChatOpenAI(
+        model=os.getenv("OPEN_AI_CHAT_MODEL", "gpt-4o-mini"),
+        temperature=0.3
+    )
+    
+    # 4. System prompt
+    system_prompt = """You are an HR assistant for nonprofit organizations in Canada. 
+Use the following context to answer questions accurately and helpfully.
+
+IMPORTANT DISCLAIMERS:
+- This tool provides general HR information only
+- Not a substitute for professional legal or HR advice
+- Consult qualified professionals before implementing policies
+- Do NOT share personal information about specific individuals
+
+Context:
+{context}
+
+Provide a clear, helpful answer. If you're not certain, say so. Always remind users to consult HR/legal professionals for important decisions."""
+
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", system_prompt),
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("human", "{input}")
+    ])
+    
+    # 5. Build RAG chain using LCEL (pipe operator)
+    # This is the modern LangChain approach for better composability
+    rag_chain = (
+        {
+            "context": itemgetter("input") | retriever | format_docs,
+            "input": itemgetter("input"),
+            "chat_history": itemgetter("chat_history")
+        }
+        | prompt
+        | llm
+        | StrOutputParser()
+    )
+    
+    # 6. Add chat history with message management
+    conversational_rag_chain = RunnableWithMessageHistory(
+        rag_chain,
+        get_session_history,
+        input_messages_key="input",
+        history_messages_key="chat_history",
+    )
+    
+    return conversational_rag_chain, retriever
+
+
+def ask_question(
+    rag_chain, 
+    retriever,
+    question: str, 
+    session_id: str = "default"
+) -> Tuple[str, List[Document]]:
+    """
+    Ask a question and get answer with sources
+    
+    Args:
+        rag_chain: The RAG chain
+        retriever: The vector store retriever for getting sources
+        question: User's question
+        session_id: Session identifier for conversation history
+        
+    Returns:
+        Tuple of (answer, source_documents)
+    """
+    
+    # Get answer from conversational chain
+    answer = rag_chain.invoke(
+        {"input": question},
+        config={"configurable": {"session_id": session_id}}
+    )
+    
+    # Retrieve source documents separately for display
+    sources = retriever.invoke(question)
+    
+    return answer, sources
+
+
+# Test function
+if __name__ == "__main__":
+    print("🤖 Initializing chatbot with latest LangChain (LCEL)...")
+    rag_chain, retriever = create_rag_chain()
+    
+    print("\n✅ Ready! Enter your question (type 'quit' to exit):\n")
+    
+    session_id = "test_session"
+    
+    while True:
+        question = input("You: ")
+        if question.lower() in ['quit', 'exit', 'q']:
+            break
+        
+        try:
+            answer, sources = ask_question(rag_chain, retriever, question, session_id)
+            
+            print(f"\nBot: {answer}\n")
+            
+            if sources:
+                print("📚 Sources:")
+                for i, doc in enumerate(sources[:3], 1):
+                    source = doc.metadata.get("source", "Unknown")
+                    print(f"  {i}. {source}")
+                print()
+        except Exception as e:
+            print(f"\n❌ Error: {str(e)}")
+            print("Make sure you have uploaded some documents first.\n")

src/ingestion.py

@@ -0,0 +1,102 @@
+"""
+Document ingestion module
+Loads PDF/DOCX files and stores them in Qdrant
+"""
+
+import os
+import sys
+from pathlib import Path
+from dotenv import load_dotenv
+from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader
+from datetime import datetime
+
+# Add parent directory to path for imports
+current_dir = Path(__file__).resolve().parent
+parent_dir = current_dir.parent
+if str(parent_dir) not in sys.path:
+    sys.path.insert(0, str(parent_dir))
+
+from src.vector_store import process_and_store
+
+load_dotenv()
+
+
+def load_document(file_path: str):
+    """
+    Load PDF or DOCX document
+    
+    Args:
+        file_path: Path to the document file
+        
+    Returns:
+        List of Document objects
+    """
+    if file_path.endswith('.pdf'):
+        loader = PyPDFLoader(file_path)
+    elif file_path.endswith('.docx'):
+        loader = Docx2txtLoader(file_path)
+    else:
+        raise ValueError("Only PDF and DOCX files are supported")
+    
+    documents = loader.load()
+    return documents
+
+
+def add_metadata(documents, source_name: str, doc_type: str = "document"):
+    """
+    Add metadata to documents
+    
+    Args:
+        documents: List of Document objects
+        source_name: Source filename
+        doc_type: Type of document (document, policy, guide, etc.)
+        
+    Returns:
+        Documents with added metadata
+    """
+    for doc in documents:
+        doc.metadata["source"] = source_name
+        doc.metadata["type"] = doc_type
+        doc.metadata["upload_date"] = datetime.now().strftime("%Y-%m-%d")
+    
+    return documents
+
+
+def ingest_document(file_path: str, doc_type: str = "document") -> int:
+    """
+    Complete document ingestion pipeline
+    
+    Args:
+        file_path: Path to the document file
+        doc_type: Type of document
+        
+    Returns:
+        Number of chunks created
+    """
+    print(f"📄 Processing: {file_path}")
+    
+    # 1. Load document
+    documents = load_document(file_path)
+    print(f"   ✅ Loaded {len(documents)} pages")
+    
+    # 2. Add metadata
+    source_name = os.path.basename(file_path)
+    documents = add_metadata(documents, source_name, doc_type)
+    
+    # 3. Chunk and store (using shared function)
+    num_chunks = process_and_store(documents)
+    
+    return num_chunks
+
+
+# Test function
+if __name__ == "__main__":
+    print("🧪 Testing document ingestion...")
+    print("\nPlease place a test PDF or DOCX file in data/documents/")
+    print("Then update the file path below and run again.\n")
+    
+    # Example:
+    # test_file = "data/documents/test.pdf"
+    # if os.path.exists(test_file):
+    #     num_chunks = ingest_document(test_file)
+    #     print(f"\n🎉 Success! Processed {num_chunks} chunks")

src/scraper.py

@@ -0,0 +1,148 @@
+"""
+Web scraping module
+Scrapes web pages using Firecrawl and stores in Qdrant
+"""
+
+import os
+import sys
+from pathlib import Path
+from dotenv import load_dotenv
+from firecrawl import FirecrawlApp
+from langchain_core.documents import Document
+from datetime import datetime
+from qdrant_client import QdrantClient
+
+# Add parent directory to path for imports
+current_dir = Path(__file__).resolve().parent
+parent_dir = current_dir.parent
+if str(parent_dir) not in sys.path:
+    sys.path.insert(0, str(parent_dir))
+
+from src.vector_store import process_and_store
+
+load_dotenv()
+
+
+def check_url_exists(url: str) -> int:
+    """
+    Check if URL already exists in Qdrant
+    
+    Args:
+        url: URL to check
+        
+    Returns:
+        Number of existing chunks for this URL (0 if not found)
+    """
+    client = QdrantClient(
+        url=os.getenv("QDRANT_URL"),
+        api_key=os.getenv("QDRANT_API_KEY")
+    )
+    collection_name = os.getenv("QDRANT_COLLECTION")
+    
+    try:
+        result = client.scroll(
+            collection_name=collection_name,
+            limit=1,
+            scroll_filter={
+                "must": [{"key": "metadata.source", "match": {"value": url}}]
+            },
+            with_payload=False
+        )
+        
+        # Count total chunks for this URL
+        count_result = client.count(
+            collection_name=collection_name,
+            count_filter={
+                "must": [{"key": "metadata.source", "match": {"value": url}}]
+            }
+        )
+        return count_result.count
+    except Exception:
+        return 0
+
+
+def scrape_url(url: str) -> str:
+    """
+    Scrape webpage content using Firecrawl
+    
+    Args:
+        url: URL to scrape
+        
+    Returns:
+        Markdown content of the webpage
+    """
+    print(f"🌐 Scraping: {url}")
+    
+    app = FirecrawlApp(api_key=os.getenv("FIRECRAWL_API_KEY"))
+    result = app.scrape(url, formats=['markdown'])
+    
+    # Handle different return types
+    if hasattr(result, 'markdown'):
+        markdown_content = result.markdown
+    elif isinstance(result, dict) and 'markdown' in result:
+        markdown_content = result['markdown']
+    else:
+        raise ValueError(f"Failed to scrape - unexpected result type: {type(result)}")
+    
+    if not markdown_content:
+        raise ValueError("Failed to scrape - no content retrieved")
+    
+    return markdown_content
+
+
+def process_and_store_webpage(url: str, force: bool = False) -> int:
+    """
+    Scrape webpage and store in vector database
+    
+    Args:
+        url: URL to scrape
+        force: If True, skip duplicate check and store anyway
+        
+    Returns:
+        Number of chunks created
+        
+    Raises:
+        ValueError: If URL already exists and force=False
+    """
+    
+    # 0. Check if URL already exists
+    if not force:
+        existing_chunks = check_url_exists(url)
+        if existing_chunks > 0:
+            raise ValueError(
+                f"URL already exists with {existing_chunks} chunks. "
+                f"Use 'Delete' to remove it first, or force=True to add anyway."
+            )
+    
+    # 1. Scrape content
+    markdown_content = scrape_url(url)
+    print(f"   ✅ Scraped {len(markdown_content)} characters")
+    
+    # 2. Create document with metadata
+    doc = Document(
+        page_content=markdown_content,
+        metadata={
+            "source": url,
+            "type": "webpage",
+            "upload_date": datetime.now().strftime("%Y-%m-%d")
+        }
+    )
+    
+    # 3. Chunk and store (using shared function)
+    num_chunks = process_and_store([doc])
+    
+    return num_chunks
+
+
+# Test function
+if __name__ == "__main__":
+    print("🧪 Testing web scraper...")
+    
+    # Test with a simple webpage
+    test_url = "https://hrintervals.ca/resources/sample-policy-inclusive-and-equitable-hiring-practices/"
+    
+    try:
+        num_chunks = process_and_store_webpage(test_url)
+        print(f"\n🎉 Success! Processed {num_chunks} chunks")
+    except Exception as e:
+        print(f"\n❌ Error: {str(e)}")

src/vector_store.py

@@ -0,0 +1,134 @@
+"""
+Shared vector storage utilities
+Handles chunking and storing documents in Qdrant
+"""
+
+import os
+from dotenv import load_dotenv
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+from langchain_qdrant import QdrantVectorStore
+from qdrant_client import QdrantClient
+from langchain_core.documents import Document
+from typing import List
+
+load_dotenv()
+
+
+def get_embeddings():
+    """Get OpenAI embeddings instance"""
+    return OpenAIEmbeddings(
+        model=os.getenv("OPEN_AI_EMBEDDING_MODEL", "text-embedding-3-small")
+    )
+
+
+def get_qdrant_client():
+    """Get Qdrant client instance"""
+    return QdrantClient(
+        url=os.getenv("QDRANT_URL"),
+        api_key=os.getenv("QDRANT_API_KEY")
+    )
+
+
+def chunk_documents(
+    documents: List[Document], 
+    chunk_size: int = 1000, 
+    chunk_overlap: int = 200
+) -> List[Document]:
+    """
+    Split documents into chunks
+    
+    Args:
+        documents: List of LangChain Document objects
+        chunk_size: Maximum characters per chunk
+        chunk_overlap: Overlapping characters between chunks
+    
+    Returns:
+        List of chunked Document objects
+    """
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separators=["\n\n", "\n", ". ", " ", ""]
+    )
+    
+    chunks = text_splitter.split_documents(documents)
+    return chunks
+
+
+def store_documents(documents: List[Document]) -> tuple[int, int]:
+    """
+    Store documents in Qdrant vector database
+    
+    Args:
+        documents: List of Document objects with content and metadata
+    
+    Returns:
+        Tuple of (expected_count, actual_stored_count)
+    """
+    embeddings = get_embeddings()
+    client = get_qdrant_client()
+    collection_name = os.getenv("QDRANT_COLLECTION")
+    
+    # Get count before storing
+    try:
+        before_count = client.count(collection_name=collection_name).count
+    except Exception:
+        before_count = 0
+    
+    # Store documents
+    vectorstore = QdrantVectorStore.from_documents(
+        documents=documents,
+        embedding=embeddings,
+        url=os.getenv("QDRANT_URL"),
+        api_key=os.getenv("QDRANT_API_KEY"),
+        collection_name=collection_name
+    )
+    
+    # Verify storage by counting after
+    try:
+        after_count = client.count(collection_name=collection_name).count
+        actual_stored = after_count - before_count
+    except Exception as e:
+        print(f"   ⚠️ Warning: Could not verify storage: {str(e)}")
+        actual_stored = len(documents)  # Assume success if can't verify
+    
+    return len(documents), actual_stored
+
+
+def process_and_store(
+    documents: List[Document], 
+    chunk_size: int = 1000, 
+    chunk_overlap: int = 200
+) -> int:
+    """
+    Complete pipeline: chunk documents and store in vector database
+    
+    Args:
+        documents: List of Document objects
+        chunk_size: Maximum characters per chunk
+        chunk_overlap: Overlapping characters between chunks
+    
+    Returns:
+        Number of chunks stored
+    """
+    # 1. Chunk documents
+    chunks = chunk_documents(documents, chunk_size, chunk_overlap)
+    print(f"   ✅ Created {len(chunks)} chunks")
+    
+    # 2. Store in Qdrant with verification
+    try:
+        expected, actual_stored = store_documents(chunks)
+        
+        if actual_stored == expected:
+            print(f"   ✅ Stored {actual_stored} chunks in Qdrant")
+        elif actual_stored > 0:
+            print(f"   ⚠️ Partial storage: expected {expected}, actually stored {actual_stored}")
+        else:
+            print(f"   ❌ Storage failed: 0 chunks stored (expected {expected})")
+            
+        return actual_stored
+        
+    except Exception as e:
+        print(f"   ❌ Error storing in Qdrant: {str(e)}")
+        raise

tests/test_connections.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""
+Test all API connections with 2025 October versions
+"""
+
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+print("🧪 Testing API Connections (October 2025)...\n")
+
+# Test 1: OpenAI
+print("1️⃣ Testing OpenAI...")
+try:
+    from openai import OpenAI
+    client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+    
+    # Test embeddings
+    response = client.embeddings.create(
+        model="text-embedding-3-small",
+        input="test"
+    )
+    print("   ✅ OpenAI connected successfully!")
+    print(f"   ✅ Embeddings working (dimension: {len(response.data[0].embedding)})")
+except Exception as e:
+    print(f"   ❌ OpenAI error: {str(e)}")
+
+# Test 2: Qdrant
+print("\n2️⃣ Testing Qdrant...")
+try:
+    from qdrant_client import QdrantClient
+    client = QdrantClient(
+        url=os.getenv("QDRANT_URL"),
+        api_key=os.getenv("QDRANT_API_KEY")
+    )
+    collections = client.get_collections()
+    print(f"   ✅ Qdrant connected! Collections: {len(collections.collections)}")
+except Exception as e:
+    print(f"   ❌ Qdrant error: {str(e)}")
+
+# Test 3: Firecrawl
+print("\n3️⃣ Testing Firecrawl...")
+try:
+    from firecrawl import FirecrawlApp
+    app = FirecrawlApp(api_key=os.getenv("FIRECRAWL_API_KEY"))
+    print("   ✅ Firecrawl initialized successfully!")
+except Exception as e:
+    print(f"   ❌ Firecrawl error: {str(e)}")
+
+# Test 4: LangChain imports (LCEL)
+print("\n4️⃣ Testing LangChain with LCEL imports...")
+try:
+    import langchain
+    from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+    from langchain_qdrant import QdrantVectorStore
+    from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+    from langchain_core.runnables.history import RunnableWithMessageHistory
+    from langchain_core.output_parsers import StrOutputParser
+    from langchain_core.documents import Document
+    from operator import itemgetter
+    
+    print(f"   ✅ LangChain version: {langchain.__version__}")
+    print("   ✅ All LangChain LCEL imports successful!")
+except Exception as e:
+    print(f"   ❌ LangChain import error: {str(e)}")
+
+# Test 5: Gradio
+print("\n5️⃣ Testing Gradio...")
+try:
+    import gradio as gr
+    print(f"   ✅ Gradio version: {gr.__version__}")
+except Exception as e:
+    print(f"   ❌ Gradio error: {str(e)}")
+
+print("\n" + "="*50)
+print("🎉 Connection tests complete!")
+print("\nNext steps:")
+print("1. Upload a test document: python src/ingestion.py")
+print("2. Test the chatbot: python src/chatbot.py")
+print("3. Start the user interface: python app.py")
+print("4. Start the admin interface: python admin.py")