Upload 103 files

.env

@@ -0,0 +1,12 @@
+# Google API Configuration for RAG System
+GOOGLE_API_KEY=AIzaSyAZQN21CjLySEybT6vOYDCz4V_e85gD42k
+
+# Database Configuration
+DATABASE_PATH=./src/streamlit_app/backend/diary.db
+
+# Vector Database Configuration
+VECTOR_DB_PATH=./src/Indexingstep/diary_vector_db_enhanced
+COLLECTION_NAME=diary_entries
+
+# RAG Configuration 
+EMBEDDING_MODEL=models/embedding-001

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+clean_repo/src/streamlit_app/src/Indexingstep/user_3_vector_db/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text
+clean_repo/src/streamlit_app/temp/recorded_audio.wav filter=lfs diff=lfs merge=lfs -text
+clean_repo/VectorDB/user_1_vector_db/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text
+src/streamlit_app/src/Indexingstep/user_3_vector_db/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text
+src/streamlit_app/temp/recorded_audio.wav filter=lfs diff=lfs merge=lfs -text
+VectorDB/user_1_vector_db/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,124 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+./venv/
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+# Vector databases & generated files
+/src/Indexingstep/src/Indexingstep/diary_vector_db_enhanced/
+*.db
+*.sqlite3
+*.bin
+# OS
+.DS_Store
+Thumbs.db
+
+# Project specific
+data/
+models/
+logs/
+*.pkl
+*.model

Backend.md

@@ -0,0 +1,65 @@
+# Backend Architecture - Personal Diary Chatbot
+
+## Tổng quan Backend
+
+Backend của dự án được xây dựng trên nền tảng FastAPI, cung cấp API RESTful cho việc xử lý nhật ký, tìm kiếm và tương tác với chatbot RAG. Hệ thống được thiết kế theo kiến trúc microservices với khả năng mở rộng cao.
+
+## 🏛️ Kiến trúc tổng thể
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Frontend      │    │   API Gateway   │    │   Core Services │
+│   (Streamlit)   │◄──►│   (FastAPI)     │◄──►│   (RAG Engine)  │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                              │
+                              ▼
+                       ┌─────────────────┐
+                       │   Data Layer    │
+                       │   (Vector DB)   │
+                       └─────────────────┘
+```
+
+## 🔧 Cấu trúc thư mục Backend
+
+```
+src/
+├── rag_service/                 # FastAPI service
+│   ├── main.py                  # Main application entry point
+│   ├── __init__.py
+│   └── __pycache__/
+├── Indexingstep/                # Data processing pipeline
+│   ├── pipeline.py              # Main indexing pipeline
+│   ├── dataloading.py           # Document loading utilities
+│   ├── diary_text_splitter.py   # Text chunking logic
+│   ├── embedding_and_storing.py # Vector embedding & storage
+│   ├── database_utils.py        # Database operations
+│   └── indexing_pipeline.py     # Pipeline orchestration
+├── Retrivel_And_Generation/     # RAG core engine
+│   ├── Retrieval_And_Generator.py # Main RAG system
+│   └── __init__.py
+├── VectorDB/                    # Vector database storage
+└── streamlit_app/               # Frontend application
+    ├── backend/                 # Backend utilities for UI
+    ├── user_auth.py             # Authentication system
+    ├── rag_client.py            # RAG service client
+    └── interface.py             # Main UI interface
+```
+## 🔮 Future Enhancements
+
+### 1. Microservices Architecture
+- **User Service**: Dedicated user management
+- **Document Service**: Document processing pipeline
+- **Search Service**: Vector search optimization
+- **Chat Service**: Conversation management
+
+### 2. Advanced Features
+- **Real-time synchronization**: WebSocket support
+- **Multi-language support**: Internationalization
+- **Advanced analytics**: User behavior tracking
+- **Machine learning**: Continuous model improvement
+
+### 3. Infrastructure Improvements
+- **Kubernetes deployment**: Container orchestration
+- **Service mesh**: Istio integration
+- **Observability**: Distributed tracing
+- **Auto-scaling**: Dynamic resource allocation

Dockerfile

@@ -0,0 +1,17 @@
+# Sử dụng image Python chính thức
+FROM python:3.10-slim
+
+# Đặt thư mục làm việc trong container
+WORKDIR /app
+
+# Sao chép file requirements.txt vào container (nếu có)
+COPY requirements.txt .
+
+# Cài đặt các thư viện phụ thuộc
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Sao chép toàn bộ mã nguồn vào container
+COPY . .
+
+# Chạy ứng dụng (thay main.py bằng file chạy chính của bạn)
+CMD ["python", "app.py"]

RAG-architecture.md

@@ -0,0 +1,254 @@
+# RAG Architecture - Personal Diary Chatbot
+
+## 🏗️ Tổng quan kiến trúc RAG
+
+Kiến trúc RAG (Retrieval-Augmented Generation) trong dự án này được thiết kế để cung cấp khả năng tìm kiếm và trả lời thông minh dựa trên dữ liệu nhật ký cá nhân của người dùng.
+
+## 🔄 Luồng xử lý RAG
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Input Query   │───►│   Query         │───►│   Vector        │
+│   (User Question)│    │   Processing    │    │   Search        │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                                                       │
+                                                       ▼
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Final Answer  │◄───│   Answer        │◄───│   Context       │
+│   (Response)    │    │   Generation    │    │   Retrieval     │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+```
+
+## 📊 Chi tiết các thành phần
+
+### 1. Data Ingestion & Indexing
+
+#### 1.1 Document Loading
+- **Input formats**: PDF, DOCX, TXT
+- **Processing**: Text extraction, cleaning, normalization
+- **Output**: Structured text data
+
+#### 1.2 Text Chunking
+```python
+# Chunking strategy
+chunk_size = 1000  # characters
+chunk_overlap = 200  # characters
+chunking_method = "recursive_character_splitter"
+```
+
+#### 1.3 Embedding Generation
+- **Model**: Google Universal Sentence Encoder (USE)
+- **Vector dimension**: 512
+- **Normalization**: L2 normalization
+- **Storage**: ChromaDB vector database
+
+### 2. Vector Database Architecture
+
+#### 2.1 ChromaDB Configuration
+```python
+# Database settings
+collection_name = f"user_{user_id}_diary"
+metadata = {
+    "user_id": user_id,
+    "source": "diary_entry",
+    "date": entry_date,
+    "chunk_id": chunk_id
+}
+```
+
+#### 2.2 Index Structure
+- **Primary key**: `user_id + chunk_id`
+- **Vector index**: HNSW (Hierarchical Navigable Small World)
+- **Distance metric**: Cosine similarity
+- **Sharding**: Per-user collections
+
+### 3. Retrieval Engine
+
+#### 3.1 Query Processing
+```python
+# Query preprocessing
+def process_query(query: str):
+    # 1. Text cleaning
+    # 2. Stop word removal
+    # 3. Lemmatization
+    # 4. Query expansion
+    return processed_query
+```
+
+#### 3.2 Vector Search
+- **Search algorithm**: K-Nearest Neighbors (KNN)
+- **Top-k results**: 5-10 most relevant chunks
+- **Similarity threshold**: 0.7 (cosine similarity)
+- **Reranking**: Semantic relevance scoring
+
+#### 3.3 Context Assembly
+```python
+# Context building
+def build_context(retrieved_chunks, query):
+    # 1. Sort by relevance score
+    # 2. Remove duplicates
+    # 3. Truncate to token limit
+    # 4. Add metadata context
+    return final_context
+```
+
+### 4. Generation Engine
+
+#### 4.1 LLM Integration
+- **Primary model**: OpenAI GPT-3.5/4
+- **Fallback model**: Local model (nếu cần)
+- **Temperature**: 0.7 (balanced creativity)
+- **Max tokens**: 500 (response length)
+
+#### 4.2 Prompt Engineering
+```python
+# System prompt template
+SYSTEM_PROMPT = """
+You are a helpful AI assistant that answers questions about personal diary entries.
+Use only the provided context to answer questions.
+If the information is not in the context, say so.
+Be conversational but professional.
+"""
+```
+
+#### 4.3 Response Generation
+```python
+# Generation pipeline
+def generate_response(query, context, chat_history):
+    # 1. Build prompt with context
+    # 2. Call LLM API
+    # 3. Post-process response
+    # 4. Validate against context
+    # 5. Return final answer
+```
+
+## 🔧 Cấu hình kỹ thuật
+
+### Performance Tuning
+
+#### 1. Chunking Optimization
+- **Optimal chunk size**: 1000 characters
+- **Overlap ratio**: 20%
+- **Chunking strategy**: Recursive character splitter
+
+#### 2. Vector Search Optimization
+- **Index type**: HNSW
+- **Search parameters**: 
+  - `ef_construction`: 200
+  - `ef_search`: 100
+  - `m`: 16
+
+#### 3. Caching Strategy
+- **Query cache**: Redis (in-memory)
+- **Embedding cache**: Local file cache
+- **Response cache**: TTL-based expiration
+
+### Scalability Features
+
+#### 1. Multi-User Support
+- **User isolation**: Separate vector collections
+- **Resource management**: Per-user memory limits
+- **Concurrent access**: Async processing
+
+#### 2. Horizontal Scaling
+- **Load balancing**: Multiple RAG instances
+- **Database sharding**: User-based distribution
+- **Microservices**: Modular architecture
+
+## 📈 Monitoring & Analytics
+
+### 1. Performance Metrics
+- **Query latency**: < 2 seconds
+- **Retrieval accuracy**: > 85%
+- **Generation quality**: User satisfaction score
+- **System throughput**: Queries per second
+
+### 2. Quality Assurance
+- **Context relevance**: Similarity score tracking
+- **Answer accuracy**: Human evaluation
+- **User feedback**: Rating system
+- **A/B testing**: Model comparison
+
+## 🚀 Deployment Architecture
+
+### 1. Development Environment
+```
+┌─────────────────┐    ┌─────────────────┐
+│   Local Python │    │   Local         │
+│   Environment  │◄──►│   ChromaDB      │
+└─────────────────┘    └─────────────────┘
+```
+
+### 2. Production Environment
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Load Balancer│    │   RAG Service   │    │   Vector DB     │
+│   (Nginx)      │◄──►│   (FastAPI)     │◄──►│   (ChromaDB)    │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                              │
+                              ▼
+                       ┌─────────────────┐
+                       │   Redis Cache   │
+                       └─────────────────┘
+```
+
+## 🔒 Security & Privacy
+
+### 1. Data Protection
+- **User isolation**: Strict separation of data
+- **Encryption**: At-rest and in-transit
+- **Access control**: Role-based permissions
+- **Audit logging**: Complete access history
+
+### 2. Privacy Compliance
+- **GDPR compliance**: Data portability
+- **Data retention**: Configurable policies
+- **User consent**: Explicit permission management
+- **Data anonymization**: Optional features
+
+## 🧪 Testing Strategy
+
+### 1. Unit Testing
+- **Component testing**: Individual modules
+- **Mock testing**: External API simulation
+- **Coverage target**: > 90%
+
+### 2. Integration Testing
+- **End-to-end testing**: Complete RAG pipeline
+- **Performance testing**: Load and stress tests
+- **Security testing**: Vulnerability assessment
+
+
+## 📚 Best Practices
+
+### 1. Model Selection
+- **Embedding models**: Domain-specific fine-tuning
+- **LLM selection**: Cost-performance balance
+- **Fallback strategies**: Graceful degradation
+
+### 2. Data Quality
+- **Input validation**: Strict data checking
+- **Cleaning pipeline**: Automated preprocessing
+- **Quality metrics**: Continuous monitoring
+
+### 3. Error Handling
+- **Graceful failures**: User-friendly error messages
+- **Retry mechanisms**: Automatic recovery
+- **Logging**: Comprehensive error tracking
+
+## 🔮 Future Enhancements
+
+### 1. Advanced Features
+- **Multi-modal RAG**: Image and text processing
+- **Temporal reasoning**: Time-based queries
+- **Emotional analysis**: Sentiment-aware responses
+
+### 2. Performance Improvements
+- **Vector quantization**: Reduced memory usage
+- **Approximate search**: Faster retrieval
+- **Model distillation**: Smaller, faster models
+
+### 3. Integration Capabilities
+- **API ecosystem**: Third-party integrations
+- **Mobile applications**: Native mobile support
+- **Voice interface**: Speech-to-text integration

README.md

@@ -1,11 +1,310 @@
----
-title: Diary Chatbot
-emoji: 🐠
-colorFrom: green
-colorTo: blue
-sdk: docker
-pinned: false
-short_description: An Diary chat bot with AI power using RAG technique.
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# RAG Personal Diary Chatbot
+
+## 📖 Project Description
+
+RAG Personal Diary Chatbot is an intelligent chatbot application that uses RAG (Retrieval-Augmented Generation) architecture to interact with users' personal diaries. The application allows users to ask questions about diary content and receive accurate answers based on actual data.
+
+## ✨ Key Features
+
+
+## 🏗️ System Architecture
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Streamlit UI  │    │   FastAPI       │    │   Vector        │
+│   (Frontend)    │◄──►│   Backend       │◄──►│   Database     │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                              │
+                              ▼
+                       ┌─────────────────┐
+                       │   RAG Engine    │
+                       │   (LLM +       │
+                       │   Retrieval)    │
+                       └─────────────────┘
+```
+
+## 🚀 Installation and Setup
+
+### System Requirements
+
+### Install Dependencies
+```bash
+# Create virtual environment
+python -m venv .venv
+
+# Activate virtual environment
+# Windows
+.venv\Scripts\activate
+# Linux/Mac
+source .venv/bin/activate
+
+# Install packages
+pip install -r requirements.txt
+```
+
+### Environment Configuration
+
+Create a `.env` file in the project root directory with the following structure:
+
+```env
+# API Keys
+OPENAI_API_KEY=your_openai_api_key_here
+GOOGLE_API_KEY=your_google_api_key_here
+ANTHROPIC_API_KEY=your_anthropic_api_key_here
+
+# Database Configuration
+DATABASE_URL=sqlite:///./user_database/auth.db
+VECTOR_DB_PATH=./VectorDB
+
+# Model Configuration
+EMBEDDING_MODEL=google-universal-sentence-encoder
+LLM_MODEL=gpt-3.5-turbo
+CHUNK_SIZE=1000
+CHUNK_OVERLAP=200
+
+# Server Configuration
+RAG_SERVICE_PORT=8001
+STREAMLIT_PORT=8501
+FASTAPI_PORT=8000
+
+# Security
+SECRET_KEY=your_secret_key_here
+JWT_SECRET=your_jwt_secret_here
+
+# Logging
+LOG_LEVEL=INFO
+LOG_FILE=./logs/app.log
+
+# Vector Database
+CHROMA_DB_PATH=./VectorDB
+PERSIST_DIRECTORY=./VectorDB
+
+# File Processing
+SUPPORTED_FORMATS=pdf,docx,txt,md
+MAX_FILE_SIZE=10485760
+TEMP_DIR=./temp
+
+# RAG Configuration
+TOP_K_RESULTS=5
+SIMILARITY_THRESHOLD=0.7
+MAX_TOKENS=4096
+TEMPERATURE=0.7
+```
+
+**Important Notes:**
+
+### Run the Application
+
+#### 1. Start RAG Service
+```bash
+python start_rag_service.py
+```
+Service will run at: http://127.0.0.1:8001
+
+#### 2. Start Streamlit UI
+```bash
+cd src/streamlit_app
+streamlit run interface.py
+```
+UI will run at: http://localhost:8501
+
+## 📁 Directory Structure
+
+```
+RAG-Personal-Diary-Chatbot/
+├── src/
+│   ├── Indexingstep/          # Data indexing pipeline
+│   ├── Retrivel_And_Generation/  # RAG engine
+│   ├── rag_service/           # FastAPI backend
+│   ├── streamlit_app/         # User interface
+│   └── VectorDB/              # Vector database
+├── notebook/                   # Jupyter notebooks
+├── tests/                      # Unit tests
+├── images/                     # Documentation images
+├── start_rag_service.py       # Service startup script
+├── .env                       # Environment variables (create from template)
+├── env_template.txt           # Environment variables template
+└── README.md
+```
+
+## 🔧 Configuration
+
+### Vector Database
+
+### AI Models
+
+## 📊 Performance
+
+
+## 🧪 Testing
+
+```bash
+# Run all tests
+python -m pytest tests/
+
+# Run specific test
+python -m pytest tests/test_rag_system.py
+```
+
+## 🤝 Contributing
+
+1. Fork the project
+2. Create feature branch (`git checkout -b feature/AmazingFeature`)
+3. Commit changes (`git commit -m 'Add some AmazingFeature'`)
+4. Push to branch (`git push origin feature/AmazingFeature`)
+5. Open Pull Request
+
+## 📝 License
+
+This project is distributed under the MIT License. See the `LICENSE` file for more details.
+
+## 📞 Contact
+
+
+## 🙏 Acknowledgments
+
+
+## 📖 Project Description
+
+RAG Personal Diary Chatbot is an intelligent chatbot application that leverages Retrieval-Augmented Generation (RAG) architecture to interact with users' personal diaries. Users can ask questions about their diary content and receive accurate, context-based answers.
+
+## ✨ Key Features
+
+- **Diary Indexing**: Automatically processes and indexes diary files (PDF, DOCX, TXT)
+- **Semantic Search**: Uses a vector database for semantic search
+- **AI Chatbot**: Natural interaction with diary data
+- **User Isolation**: Each user has a separate vector database
+- **Web Interface**: Easy-to-use Streamlit UI
+- **REST API**: FastAPI backend for integration
+
+## 🏗️ System Architecture
+
+```
+┌───────────────┐    ┌───────────────┐    ┌───────────────┐
+│ Streamlit UI  │◄──►│   FastAPI     │◄──►│  Vector DB    │
+│ (Frontend)    │    │   Backend     │    │  (ChromaDB)   │
+└───────────────┘    └───────────────┘    └───────────────┘
+                              │
+                              ▼
+                       ┌───────────────┐
+                       │   RAG Engine  │
+                       │ (LLM +        │
+                       │  Retrieval)   │
+                       └───────────────┘
+```
+
+## 🚀 Installation and Setup
+
+### System Requirements
+
+- Python 3.8+
+
+### Install Dependencies
+
+```bash
+# Create virtual environment
+python -m venv .venv
+
+# Activate virtual environment
+# Windows
+.venv\Scripts\activate
+# Linux/Mac
+source .venv/bin/activate
+
+# Install packages
+pip install -r requirements.txt
+```
+
+### Environment Configuration
+
+Create a `.env` file in the project root directory with the following structure:
+
+```env
+# Google API Configuration for RAG System
+GOOGLE_API_KEY=[Google API key]
+
+# Database Configuration
+DATABASE_PATH=./src/streamlit_app/backend/diary.db
+
+# Vector Database Configuration
+VECTOR_DB_PATH=./src/Indexingstep/diary_vector_db_enhanced
+COLLECTION_NAME=diary_entries
+
+# RAG Configuration
+EMBEDDING_MODEL=models/embedding-001
+CHAT_MODEL=gemini-2.5-flash
+```
+
+**Important Notes:**
+- Replace all placeholder values with your actual API keys and configuration
+- Keep your `.env` file secure and never commit it to version control
+- The `.env` file is already included in `.gitignore`
+- Use `env_template.txt` as a reference to create your `.env` file
+
+### Run the Application
+
+```bash
+# Start the RAG backend service
+python start_rag_service.py
+
+# Start the Streamlit UI
+streamlit run src/streamlit_app/interface.py
+```
+
+## 📁 Directory Structure
+
+```
+RAG-Personal-Diary-Chatbot/
+├── src/
+│   ├── Indexingstep/          # Data indexing pipeline
+│   ├── Retrivel_And_Generation/  # RAG engine
+│   ├── rag_service/           # FastAPI backend
+│   ├── streamlit_app/         # User interface
+│   └── VectorDB/              # Vector database
+├── notebook/                  # Jupyter notebooks
+├── tests/                     # Unit tests
+├── images/                    # Documentation images
+├── start_rag_service.py       # Service startup script
+├── .env                       # Environment variables (create from template)
+├── env_template.txt           # Environment variables template
+└── README.md
+```
+
+## 🔧 Configuration
+
+### Vector Database
+- **ChromaDB**: Main database for vector embeddings
+- **Chunk size**: 1000 characters (customizable)
+- **Overlap**: 200 characters between chunks
+
+### AI Models
+- **Embedding**: Google's Universal Sentence Encoder
+- **LLM**: Google Gemini (can be replaced with other models)
+
+## 📊 Performance
+
+- **Processing time**: ~2-5 seconds per question
+- **Accuracy**: 85-95% depending on data quality
+- **Scalability**: Supports thousands of diaries
+
+
+## 🤝 Contributing
+
+1. Fork the project
+2. Create a feature branch (`git checkout -b feature/AmazingFeature`)
+3. Commit your changes (`git commit -m 'Add some AmazingFeature'`)
+4. Push to the branch (`git push origin feature/AmazingFeature`)
+5. Open a Pull Request
+## 📞 Contact
+
+- **Author**: [DongAnh]
+- **Email**: [donganhng098@gmail.com]
+- **GitHub**: [github.com/DongAnh]
+
+## 🙏 Acknowledgments
+
+- Gemini for GPT models
+- Google for Universal Sentence Encoder
+- ChromaDB team for vector database
+- FastAPI and Streamlit communities
+- RAG architecture

VectorDB/user_1_vector_db/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e895be71c0752f44743806cd7439624649d5e41500865e79c77e07c6d1dca87
+size 163840

app.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""
+Start RAG Service for Personal Diary Chatbot
+"""
+import subprocess
+import sys
+import os
+from pathlib import Path
+import time
+
+def check_requirements():
+    """Check if required packages are installed."""
+    required_packages = ['fastapi', 'uvicorn']
+    missing_packages = []
+    
+    for package in required_packages:
+        try:
+            __import__(package)
+        except ImportError:
+            missing_packages.append(package)
+    
+    if missing_packages:
+        print(f"❌ Missing packages: {', '.join(missing_packages)}")
+        print(f"Install with: pip install {' '.join(missing_packages)}")
+        return False
+    
+    return True
+
+def setup_environment():
+    """Setup environment and directories."""
+    # Ensure VectorDB directory exists
+    vector_db_dir = Path("src/VectorDB")
+    vector_db_dir.mkdir(parents=True, exist_ok=True)
+    print(f"📁 Vector DB directory: {vector_db_dir.absolute()}")
+    
+    # Check for .env file
+    env_file = Path("src/Indexingstep/.env")
+    if env_file.exists():
+        print(f"✅ Environment file found: {env_file}")
+    else:
+        print(f"⚠️  Environment file not found: {env_file}")
+        print("Make sure GOOGLE_API_KEY is set in environment")
+
+def start_service():
+    """Start the RAG FastAPI service."""
+    if not check_requirements():
+        return
+    
+    setup_environment()
+    
+    service_file = Path("src/rag_service/main.py")
+    
+    if not service_file.exists():
+        print(f"❌ Service file not found: {service_file}")
+        print("Please create the RAG service file first")
+        return
+    
+    print("🚀 Starting RAG Service...")
+    print("📍 Service URL: http://0.0.0.0:8001")
+    print("📖 API Docs: http://0.0.0.0:8001/docs")
+    print("💾 Vector databases will be stored in: src/VectorDB/")
+    print("\nPress Ctrl+C to stop the service")
+    print("-" * 50)
+    
+    try:
+        # Change to project root directory
+        os.chdir(Path(__file__).parent)
+        
+        # Start the service in the background
+        process = subprocess.Popen([
+            sys.executable, "-m", "uvicorn",
+            "src.rag_service.main:app",
+            "--host", "0.0.0.0",
+            "--port", "8001",
+            "--reload"
+        ])
+        print(f"🔄 RAG Service running in background (PID: {process.pid})")
+        return process
+    except Exception as e:
+        print(f"❌ Error starting service: {e}")
+        return None
+
+def start_streamlit():
+    # Start Streamlit UI on port 7860 (default for Spaces)
+    os.system("streamlit run src/streamlit_app/interface.py --server.port 7860")
+
+if __name__ == "__main__":
+    start_service()
+    time.sleep(3)
+    start_streamlit()

clean_repo/.env

@@ -0,0 +1,12 @@
+# Google API Configuration for RAG System
+GOOGLE_API_KEY=AIzaSyAZQN21CjLySEybT6vOYDCz4V_e85gD42k
+
+# Database Configuration
+DATABASE_PATH=./src/streamlit_app/backend/diary.db
+
+# Vector Database Configuration
+VECTOR_DB_PATH=./src/Indexingstep/diary_vector_db_enhanced
+COLLECTION_NAME=diary_entries
+
+# RAG Configuration 
+EMBEDDING_MODEL=models/embedding-001

clean_repo/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

clean_repo/.gitignore

@@ -0,0 +1,128 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+./venv/
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project specific
+data/
+models/
+logs/
+*.pkl
+*.model
+
+# Vector databases & generated files
+/VectorDB/
+/src/VectorDB/
+/src/Indexingstep/src/Indexingstep/diary_vector_db_enhanced/
+*.db
+*.sqlite3
+*.bin

clean_repo/Backend.md

@@ -0,0 +1,65 @@
+# Backend Architecture - Personal Diary Chatbot
+
+## Tổng quan Backend
+
+Backend của dự án được xây dựng trên nền tảng FastAPI, cung cấp API RESTful cho việc xử lý nhật ký, tìm kiếm và tương tác với chatbot RAG. Hệ thống được thiết kế theo kiến trúc microservices với khả năng mở rộng cao.
+
+## 🏛️ Kiến trúc tổng thể
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Frontend      │    │   API Gateway   │    │   Core Services │
+│   (Streamlit)   │◄──►│   (FastAPI)     │◄──►│   (RAG Engine)  │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                              │
+                              ▼
+                       ┌─────────────────┐
+                       │   Data Layer    │
+                       │   (Vector DB)   │
+                       └─────────────────┘
+```
+
+## 🔧 Cấu trúc thư mục Backend
+
+```
+src/
+├── rag_service/                 # FastAPI service
+│   ├── main.py                  # Main application entry point
+│   ├── __init__.py
+│   └── __pycache__/
+├── Indexingstep/                # Data processing pipeline
+│   ├── pipeline.py              # Main indexing pipeline
+│   ├── dataloading.py           # Document loading utilities
+│   ├── diary_text_splitter.py   # Text chunking logic
+│   ├── embedding_and_storing.py # Vector embedding & storage
+│   ├── database_utils.py        # Database operations
+│   └── indexing_pipeline.py     # Pipeline orchestration
+├── Retrivel_And_Generation/     # RAG core engine
+│   ├── Retrieval_And_Generator.py # Main RAG system
+│   └── __init__.py
+├── VectorDB/                    # Vector database storage
+└── streamlit_app/               # Frontend application
+    ├── backend/                 # Backend utilities for UI
+    ├── user_auth.py             # Authentication system
+    ├── rag_client.py            # RAG service client
+    └── interface.py             # Main UI interface
+```
+## 🔮 Future Enhancements
+
+### 1. Microservices Architecture
+- **User Service**: Dedicated user management
+- **Document Service**: Document processing pipeline
+- **Search Service**: Vector search optimization
+- **Chat Service**: Conversation management
+
+### 2. Advanced Features
+- **Real-time synchronization**: WebSocket support
+- **Multi-language support**: Internationalization
+- **Advanced analytics**: User behavior tracking
+- **Machine learning**: Continuous model improvement
+
+### 3. Infrastructure Improvements
+- **Kubernetes deployment**: Container orchestration
+- **Service mesh**: Istio integration
+- **Observability**: Distributed tracing
+- **Auto-scaling**: Dynamic resource allocation

clean_repo/Dockerfile

@@ -0,0 +1,17 @@
+# Sử dụng image Python chính thức
+FROM python:3.10-slim
+
+# Đặt thư mục làm việc trong container
+WORKDIR /app
+
+# Sao chép file requirements.txt vào container (nếu có)
+COPY requirements.txt .
+
+# Cài đặt các thư viện phụ thuộc
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Sao chép toàn bộ mã nguồn vào container
+COPY . .
+
+# Chạy ứng dụng (thay main.py bằng file chạy chính của bạn)
+CMD ["python", "app.py"]

clean_repo/RAG-architecture.md

@@ -0,0 +1,254 @@
+# RAG Architecture - Personal Diary Chatbot
+
+## 🏗️ Tổng quan kiến trúc RAG
+
+Kiến trúc RAG (Retrieval-Augmented Generation) trong dự án này được thiết kế để cung cấp khả năng tìm kiếm và trả lời thông minh dựa trên dữ liệu nhật ký cá nhân của người dùng.
+
+## 🔄 Luồng xử lý RAG
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Input Query   │───►│   Query         │───►│   Vector        │
+│   (User Question)│    │   Processing    │    │   Search        │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                                                       │
+                                                       ▼
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Final Answer  │◄───│   Answer        │◄───│   Context       │
+│   (Response)    │    │   Generation    │    │   Retrieval     │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+```
+
+## 📊 Chi tiết các thành phần
+
+### 1. Data Ingestion & Indexing
+
+#### 1.1 Document Loading
+- **Input formats**: PDF, DOCX, TXT
+- **Processing**: Text extraction, cleaning, normalization
+- **Output**: Structured text data
+
+#### 1.2 Text Chunking
+```python
+# Chunking strategy
+chunk_size = 1000  # characters
+chunk_overlap = 200  # characters
+chunking_method = "recursive_character_splitter"
+```
+
+#### 1.3 Embedding Generation
+- **Model**: Google Universal Sentence Encoder (USE)
+- **Vector dimension**: 512
+- **Normalization**: L2 normalization
+- **Storage**: ChromaDB vector database
+
+### 2. Vector Database Architecture
+
+#### 2.1 ChromaDB Configuration
+```python
+# Database settings
+collection_name = f"user_{user_id}_diary"
+metadata = {
+    "user_id": user_id,
+    "source": "diary_entry",
+    "date": entry_date,
+    "chunk_id": chunk_id
+}
+```
+
+#### 2.2 Index Structure
+- **Primary key**: `user_id + chunk_id`
+- **Vector index**: HNSW (Hierarchical Navigable Small World)
+- **Distance metric**: Cosine similarity
+- **Sharding**: Per-user collections
+
+### 3. Retrieval Engine
+
+#### 3.1 Query Processing
+```python
+# Query preprocessing
+def process_query(query: str):
+    # 1. Text cleaning
+    # 2. Stop word removal
+    # 3. Lemmatization
+    # 4. Query expansion
+    return processed_query
+```
+
+#### 3.2 Vector Search
+- **Search algorithm**: K-Nearest Neighbors (KNN)
+- **Top-k results**: 5-10 most relevant chunks
+- **Similarity threshold**: 0.7 (cosine similarity)
+- **Reranking**: Semantic relevance scoring
+
+#### 3.3 Context Assembly
+```python
+# Context building
+def build_context(retrieved_chunks, query):
+    # 1. Sort by relevance score
+    # 2. Remove duplicates
+    # 3. Truncate to token limit
+    # 4. Add metadata context
+    return final_context
+```
+
+### 4. Generation Engine
+
+#### 4.1 LLM Integration
+- **Primary model**: OpenAI GPT-3.5/4
+- **Fallback model**: Local model (nếu cần)
+- **Temperature**: 0.7 (balanced creativity)
+- **Max tokens**: 500 (response length)
+
+#### 4.2 Prompt Engineering
+```python
+# System prompt template
+SYSTEM_PROMPT = """
+You are a helpful AI assistant that answers questions about personal diary entries.
+Use only the provided context to answer questions.
+If the information is not in the context, say so.
+Be conversational but professional.
+"""
+```
+
+#### 4.3 Response Generation
+```python
+# Generation pipeline
+def generate_response(query, context, chat_history):
+    # 1. Build prompt with context
+    # 2. Call LLM API
+    # 3. Post-process response
+    # 4. Validate against context
+    # 5. Return final answer
+```
+
+## 🔧 Cấu hình kỹ thuật
+
+### Performance Tuning
+
+#### 1. Chunking Optimization
+- **Optimal chunk size**: 1000 characters
+- **Overlap ratio**: 20%
+- **Chunking strategy**: Recursive character splitter
+
+#### 2. Vector Search Optimization
+- **Index type**: HNSW
+- **Search parameters**: 
+  - `ef_construction`: 200
+  - `ef_search`: 100
+  - `m`: 16
+
+#### 3. Caching Strategy
+- **Query cache**: Redis (in-memory)
+- **Embedding cache**: Local file cache
+- **Response cache**: TTL-based expiration
+
+### Scalability Features
+
+#### 1. Multi-User Support
+- **User isolation**: Separate vector collections
+- **Resource management**: Per-user memory limits
+- **Concurrent access**: Async processing
+
+#### 2. Horizontal Scaling
+- **Load balancing**: Multiple RAG instances
+- **Database sharding**: User-based distribution
+- **Microservices**: Modular architecture
+
+## 📈 Monitoring & Analytics
+
+### 1. Performance Metrics
+- **Query latency**: < 2 seconds
+- **Retrieval accuracy**: > 85%
+- **Generation quality**: User satisfaction score
+- **System throughput**: Queries per second
+
+### 2. Quality Assurance
+- **Context relevance**: Similarity score tracking
+- **Answer accuracy**: Human evaluation
+- **User feedback**: Rating system
+- **A/B testing**: Model comparison
+
+## 🚀 Deployment Architecture
+
+### 1. Development Environment
+```
+┌─────────────────┐    ┌─────────────────┐
+│   Local Python │    │   Local         │
+│   Environment  │◄──►│   ChromaDB      │
+└─────────────────┘    └─────────────────┘
+```
+
+### 2. Production Environment
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Load Balancer│    │   RAG Service   │    │   Vector DB     │
+│   (Nginx)      │◄──►│   (FastAPI)     │◄──►│   (ChromaDB)    │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                              │
+                              ▼
+                       ┌─────────────────┐
+                       │   Redis Cache   │
+                       └─────────────────┘
+```
+
+## 🔒 Security & Privacy
+
+### 1. Data Protection
+- **User isolation**: Strict separation of data
+- **Encryption**: At-rest and in-transit
+- **Access control**: Role-based permissions
+- **Audit logging**: Complete access history
+
+### 2. Privacy Compliance
+- **GDPR compliance**: Data portability
+- **Data retention**: Configurable policies
+- **User consent**: Explicit permission management
+- **Data anonymization**: Optional features
+
+## 🧪 Testing Strategy
+
+### 1. Unit Testing
+- **Component testing**: Individual modules
+- **Mock testing**: External API simulation
+- **Coverage target**: > 90%
+
+### 2. Integration Testing
+- **End-to-end testing**: Complete RAG pipeline
+- **Performance testing**: Load and stress tests
+- **Security testing**: Vulnerability assessment
+
+
+## 📚 Best Practices
+
+### 1. Model Selection
+- **Embedding models**: Domain-specific fine-tuning
+- **LLM selection**: Cost-performance balance
+- **Fallback strategies**: Graceful degradation
+
+### 2. Data Quality
+- **Input validation**: Strict data checking
+- **Cleaning pipeline**: Automated preprocessing
+- **Quality metrics**: Continuous monitoring
+
+### 3. Error Handling
+- **Graceful failures**: User-friendly error messages
+- **Retry mechanisms**: Automatic recovery
+- **Logging**: Comprehensive error tracking
+
+## 🔮 Future Enhancements
+
+### 1. Advanced Features
+- **Multi-modal RAG**: Image and text processing
+- **Temporal reasoning**: Time-based queries
+- **Emotional analysis**: Sentiment-aware responses
+
+### 2. Performance Improvements
+- **Vector quantization**: Reduced memory usage
+- **Approximate search**: Faster retrieval
+- **Model distillation**: Smaller, faster models
+
+### 3. Integration Capabilities
+- **API ecosystem**: Third-party integrations
+- **Mobile applications**: Native mobile support
+- **Voice interface**: Speech-to-text integration

clean_repo/README.md

@@ -0,0 +1,310 @@
+# RAG Personal Diary Chatbot
+
+## 📖 Project Description
+
+RAG Personal Diary Chatbot is an intelligent chatbot application that uses RAG (Retrieval-Augmented Generation) architecture to interact with users' personal diaries. The application allows users to ask questions about diary content and receive accurate answers based on actual data.
+
+## ✨ Key Features
+
+
+## 🏗️ System Architecture
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Streamlit UI  │    │   FastAPI       │    │   Vector        │
+│   (Frontend)    │◄──►│   Backend       │◄──►│   Database     │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                              │
+                              ▼
+                       ┌─────────────────┐
+                       │   RAG Engine    │
+                       │   (LLM +       │
+                       │   Retrieval)    │
+                       └─────────────────┘
+```
+
+## 🚀 Installation and Setup
+
+### System Requirements
+
+### Install Dependencies
+```bash
+# Create virtual environment
+python -m venv .venv
+
+# Activate virtual environment
+# Windows
+.venv\Scripts\activate
+# Linux/Mac
+source .venv/bin/activate
+
+# Install packages
+pip install -r requirements.txt
+```
+
+### Environment Configuration
+
+Create a `.env` file in the project root directory with the following structure:
+
+```env
+# API Keys
+OPENAI_API_KEY=your_openai_api_key_here
+GOOGLE_API_KEY=your_google_api_key_here
+ANTHROPIC_API_KEY=your_anthropic_api_key_here
+
+# Database Configuration
+DATABASE_URL=sqlite:///./user_database/auth.db
+VECTOR_DB_PATH=./VectorDB
+
+# Model Configuration
+EMBEDDING_MODEL=google-universal-sentence-encoder
+LLM_MODEL=gpt-3.5-turbo
+CHUNK_SIZE=1000
+CHUNK_OVERLAP=200
+
+# Server Configuration
+RAG_SERVICE_PORT=8001
+STREAMLIT_PORT=8501
+FASTAPI_PORT=8000
+
+# Security
+SECRET_KEY=your_secret_key_here
+JWT_SECRET=your_jwt_secret_here
+
+# Logging
+LOG_LEVEL=INFO
+LOG_FILE=./logs/app.log
+
+# Vector Database
+CHROMA_DB_PATH=./VectorDB
+PERSIST_DIRECTORY=./VectorDB
+
+# File Processing
+SUPPORTED_FORMATS=pdf,docx,txt,md
+MAX_FILE_SIZE=10485760
+TEMP_DIR=./temp
+
+# RAG Configuration
+TOP_K_RESULTS=5
+SIMILARITY_THRESHOLD=0.7
+MAX_TOKENS=4096
+TEMPERATURE=0.7
+```
+
+**Important Notes:**
+
+### Run the Application
+
+#### 1. Start RAG Service
+```bash
+python start_rag_service.py
+```
+Service will run at: http://127.0.0.1:8001
+
+#### 2. Start Streamlit UI
+```bash
+cd src/streamlit_app
+streamlit run interface.py
+```
+UI will run at: http://localhost:8501
+
+## 📁 Directory Structure
+
+```
+RAG-Personal-Diary-Chatbot/
+├── src/
+│   ├── Indexingstep/          # Data indexing pipeline
+│   ├── Retrivel_And_Generation/  # RAG engine
+│   ├── rag_service/           # FastAPI backend
+│   ├── streamlit_app/         # User interface
+│   └── VectorDB/              # Vector database
+├── notebook/                   # Jupyter notebooks
+├── tests/                      # Unit tests
+├── images/                     # Documentation images
+├── start_rag_service.py       # Service startup script
+├── .env                       # Environment variables (create from template)
+├── env_template.txt           # Environment variables template
+└── README.md
+```
+
+## 🔧 Configuration
+
+### Vector Database
+
+### AI Models
+
+## 📊 Performance
+
+
+## 🧪 Testing
+
+```bash
+# Run all tests
+python -m pytest tests/
+
+# Run specific test
+python -m pytest tests/test_rag_system.py
+```
+
+## 🤝 Contributing
+
+1. Fork the project
+2. Create feature branch (`git checkout -b feature/AmazingFeature`)
+3. Commit changes (`git commit -m 'Add some AmazingFeature'`)
+4. Push to branch (`git push origin feature/AmazingFeature`)
+5. Open Pull Request
+
+## 📝 License
+
+This project is distributed under the MIT License. See the `LICENSE` file for more details.
+
+## 📞 Contact
+
+
+## 🙏 Acknowledgments
+
+
+## 📖 Project Description
+
+RAG Personal Diary Chatbot is an intelligent chatbot application that leverages Retrieval-Augmented Generation (RAG) architecture to interact with users' personal diaries. Users can ask questions about their diary content and receive accurate, context-based answers.
+
+## ✨ Key Features
+
+- **Diary Indexing**: Automatically processes and indexes diary files (PDF, DOCX, TXT)
+- **Semantic Search**: Uses a vector database for semantic search
+- **AI Chatbot**: Natural interaction with diary data
+- **User Isolation**: Each user has a separate vector database
+- **Web Interface**: Easy-to-use Streamlit UI
+- **REST API**: FastAPI backend for integration
+
+## 🏗️ System Architecture
+
+```
+┌───────────────┐    ┌───────────────┐    ┌───────────────┐
+│ Streamlit UI  │◄──►│   FastAPI     │◄──►│  Vector DB    │
+│ (Frontend)    │    │   Backend     │    │  (ChromaDB)   │
+└───────────────┘    └───────────────┘    └───────────────┘
+                              │
+                              ▼
+                       ┌───────────────┐
+                       │   RAG Engine  │
+                       │ (LLM +        │
+                       │  Retrieval)   │
+                       └───────────────┘
+```
+
+## 🚀 Installation and Setup
+
+### System Requirements
+
+- Python 3.8+
+
+### Install Dependencies
+
+```bash
+# Create virtual environment
+python -m venv .venv
+
+# Activate virtual environment
+# Windows
+.venv\Scripts\activate
+# Linux/Mac
+source .venv/bin/activate
+
+# Install packages
+pip install -r requirements.txt
+```
+
+### Environment Configuration
+
+Create a `.env` file in the project root directory with the following structure:
+
+```env
+# Google API Configuration for RAG System
+GOOGLE_API_KEY=[Google API key]
+
+# Database Configuration
+DATABASE_PATH=./src/streamlit_app/backend/diary.db
+
+# Vector Database Configuration
+VECTOR_DB_PATH=./src/Indexingstep/diary_vector_db_enhanced
+COLLECTION_NAME=diary_entries
+
+# RAG Configuration
+EMBEDDING_MODEL=models/embedding-001
+CHAT_MODEL=gemini-2.5-flash
+```
+
+**Important Notes:**
+- Replace all placeholder values with your actual API keys and configuration
+- Keep your `.env` file secure and never commit it to version control
+- The `.env` file is already included in `.gitignore`
+- Use `env_template.txt` as a reference to create your `.env` file
+
+### Run the Application
+
+```bash
+# Start the RAG backend service
+python start_rag_service.py
+
+# Start the Streamlit UI
+streamlit run src/streamlit_app/interface.py
+```
+
+## 📁 Directory Structure
+
+```
+RAG-Personal-Diary-Chatbot/
+├── src/
+│   ├── Indexingstep/          # Data indexing pipeline
+│   ├── Retrivel_And_Generation/  # RAG engine
+│   ├── rag_service/           # FastAPI backend
+│   ├── streamlit_app/         # User interface
+│   └── VectorDB/              # Vector database
+├── notebook/                  # Jupyter notebooks
+├── tests/                     # Unit tests
+├── images/                    # Documentation images
+├── start_rag_service.py       # Service startup script
+├── .env                       # Environment variables (create from template)
+├── env_template.txt           # Environment variables template
+└── README.md
+```
+
+## 🔧 Configuration
+
+### Vector Database
+- **ChromaDB**: Main database for vector embeddings
+- **Chunk size**: 1000 characters (customizable)
+- **Overlap**: 200 characters between chunks
+
+### AI Models
+- **Embedding**: Google's Universal Sentence Encoder
+- **LLM**: Google Gemini (can be replaced with other models)
+
+## 📊 Performance
+
+- **Processing time**: ~2-5 seconds per question
+- **Accuracy**: 85-95% depending on data quality
+- **Scalability**: Supports thousands of diaries
+
+
+## 🤝 Contributing
+
+1. Fork the project
+2. Create a feature branch (`git checkout -b feature/AmazingFeature`)
+3. Commit your changes (`git commit -m 'Add some AmazingFeature'`)
+4. Push to the branch (`git push origin feature/AmazingFeature`)
+5. Open a Pull Request
+## 📞 Contact
+
+- **Author**: [huytrao]
+- **Email**: [traohuy098@gmail.com]
+- **GitHub**: [github.com/huytrao]
+
+## 🙏 Acknowledgments
+
+- Gemini for GPT models
+- Google for Universal Sentence Encoder
+- ChromaDB team for vector database
+- FastAPI and Streamlit communities
+- RAG architecture

clean_repo/VectorDB/user_1_vector_db/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e895be71c0752f44743806cd7439624649d5e41500865e79c77e07c6d1dca87
+size 163840

clean_repo/app.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""
+Start RAG Service for Personal Diary Chatbot
+"""
+import subprocess
+import sys
+import os
+from pathlib import Path
+import time
+
+def check_requirements():
+    """Check if required packages are installed."""
+    required_packages = ['fastapi', 'uvicorn']
+    missing_packages = []
+    
+    for package in required_packages:
+        try:
+            __import__(package)
+        except ImportError:
+            missing_packages.append(package)
+    
+    if missing_packages:
+        print(f"❌ Missing packages: {', '.join(missing_packages)}")
+        print(f"Install with: pip install {' '.join(missing_packages)}")
+        return False
+    
+    return True
+
+def setup_environment():
+    """Setup environment and directories."""
+    # Ensure VectorDB directory exists
+    vector_db_dir = Path("src/VectorDB")
+    vector_db_dir.mkdir(parents=True, exist_ok=True)
+    print(f"📁 Vector DB directory: {vector_db_dir.absolute()}")
+    
+    # Check for .env file
+    env_file = Path("src/Indexingstep/.env")
+    if env_file.exists():
+        print(f"✅ Environment file found: {env_file}")
+    else:
+        print(f"⚠️  Environment file not found: {env_file}")
+        print("Make sure GOOGLE_API_KEY is set in environment")
+
+def start_service():
+    """Start the RAG FastAPI service."""
+    if not check_requirements():
+        return
+    
+    setup_environment()
+    
+    service_file = Path("src/rag_service/main.py")
+    
+    if not service_file.exists():
+        print(f"❌ Service file not found: {service_file}")
+        print("Please create the RAG service file first")
+        return
+    
+    print("🚀 Starting RAG Service...")
+    print("📍 Service URL: http://0.0.0.0:8001")
+    print("📖 API Docs: http://0.0.0.0:8001/docs")
+    print("💾 Vector databases will be stored in: src/VectorDB/")
+    print("\nPress Ctrl+C to stop the service")
+    print("-" * 50)
+    
+    try:
+        # Change to project root directory
+        os.chdir(Path(__file__).parent)
+        
+        # Start the service in the background
+        process = subprocess.Popen([
+            sys.executable, "-m", "uvicorn",
+            "src.rag_service.main:app",
+            "--host", "0.0.0.0",
+            "--port", "8001",
+            "--reload"
+        ])
+        print(f"🔄 RAG Service running in background (PID: {process.pid})")
+        return process
+    except Exception as e:
+        print(f"❌ Error starting service: {e}")
+        return None
+
+def start_streamlit():
+    # Start Streamlit UI on port 7860 (default for Spaces)
+    os.system("streamlit run src/streamlit_app/interface.py --server.port 7860")
+
+if __name__ == "__main__":
+    start_service()
+    time.sleep(3)
+    start_streamlit()

clean_repo/clean_repo/.env

@@ -0,0 +1,12 @@
+# Google API Configuration for RAG System
+GOOGLE_API_KEY=AIzaSyAZQN21CjLySEybT6vOYDCz4V_e85gD42k
+
+# Database Configuration
+DATABASE_PATH=./src/streamlit_app/backend/diary.db
+
+# Vector Database Configuration
+VECTOR_DB_PATH=./src/Indexingstep/diary_vector_db_enhanced
+COLLECTION_NAME=diary_entries
+
+# RAG Configuration 
+EMBEDDING_MODEL=models/embedding-001

clean_repo/clean_repo/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

clean_repo/clean_repo/.gitignore

@@ -0,0 +1,120 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+./venv/
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project specific
+data/
+models/
+logs/
+*.pkl
+*.model

clean_repo/clean_repo/Backend.md

@@ -0,0 +1,65 @@
+# Backend Architecture - Personal Diary Chatbot
+
+## Tổng quan Backend
+
+Backend của dự án được xây dựng trên nền tảng FastAPI, cung cấp API RESTful cho việc xử lý nhật ký, tìm kiếm và tương tác với chatbot RAG. Hệ thống được thiết kế theo kiến trúc microservices với khả năng mở rộng cao.
+
+## 🏛️ Kiến trúc tổng thể
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Frontend      │    │   API Gateway   │    │   Core Services │
+│   (Streamlit)   │◄──►│   (FastAPI)     │◄──►│   (RAG Engine)  │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                              │
+                              ▼
+                       ┌─────────────────┐
+                       │   Data Layer    │
+                       │   (Vector DB)   │
+                       └─────────────────┘
+```
+
+## 🔧 Cấu trúc thư mục Backend
+
+```
+src/
+├── rag_service/                 # FastAPI service
+│   ├── main.py                  # Main application entry point
+│   ├── __init__.py
+│   └── __pycache__/
+├── Indexingstep/                # Data processing pipeline
+│   ├── pipeline.py              # Main indexing pipeline
+│   ├── dataloading.py           # Document loading utilities
+│   ├── diary_text_splitter.py   # Text chunking logic
+│   ├── embedding_and_storing.py # Vector embedding & storage
+│   ├── database_utils.py        # Database operations
+│   └── indexing_pipeline.py     # Pipeline orchestration
+├── Retrivel_And_Generation/     # RAG core engine
+│   ├── Retrieval_And_Generator.py # Main RAG system
+│   └── __init__.py
+├── VectorDB/                    # Vector database storage
+└── streamlit_app/               # Frontend application
+    ├── backend/                 # Backend utilities for UI
+    ├── user_auth.py             # Authentication system
+    ├── rag_client.py            # RAG service client
+    └── interface.py             # Main UI interface
+```
+## 🔮 Future Enhancements
+
+### 1. Microservices Architecture
+- **User Service**: Dedicated user management
+- **Document Service**: Document processing pipeline
+- **Search Service**: Vector search optimization
+- **Chat Service**: Conversation management
+
+### 2. Advanced Features
+- **Real-time synchronization**: WebSocket support
+- **Multi-language support**: Internationalization
+- **Advanced analytics**: User behavior tracking
+- **Machine learning**: Continuous model improvement
+
+### 3. Infrastructure Improvements
+- **Kubernetes deployment**: Container orchestration
+- **Service mesh**: Istio integration
+- **Observability**: Distributed tracing
+- **Auto-scaling**: Dynamic resource allocation

clean_repo/clean_repo/Dockerfile

@@ -0,0 +1,17 @@
+# Sử dụng image Python chính thức
+FROM python:3.10-slim
+
+# Đặt thư mục làm việc trong container
+WORKDIR /app
+
+# Sao chép file requirements.txt vào container (nếu có)
+COPY requirements.txt .
+
+# Cài đặt các thư viện phụ thuộc
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Sao chép toàn bộ mã nguồn vào container
+COPY . .
+
+# Chạy ứng dụng (thay main.py bằng file chạy chính của bạn)
+CMD ["python", "app.py"]

clean_repo/clean_repo/RAG-architecture.md

@@ -0,0 +1,254 @@
+# RAG Architecture - Personal Diary Chatbot
+
+## 🏗️ Tổng quan kiến trúc RAG
+
+Kiến trúc RAG (Retrieval-Augmented Generation) trong dự án này được thiết kế để cung cấp khả năng tìm kiếm và trả lời thông minh dựa trên dữ liệu nhật ký cá nhân của người dùng.
+
+## 🔄 Luồng xử lý RAG
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Input Query   │───►│   Query         │───►│   Vector        │
+│   (User Question)│    │   Processing    │    │   Search        │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                                                       │
+                                                       ▼
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Final Answer  │◄───│   Answer        │◄───│   Context       │
+│   (Response)    │    │   Generation    │    │   Retrieval     │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+```
+
+## 📊 Chi tiết các thành phần
+
+### 1. Data Ingestion & Indexing
+
+#### 1.1 Document Loading
+- **Input formats**: PDF, DOCX, TXT
+- **Processing**: Text extraction, cleaning, normalization
+- **Output**: Structured text data
+
+#### 1.2 Text Chunking
+```python
+# Chunking strategy
+chunk_size = 1000  # characters
+chunk_overlap = 200  # characters
+chunking_method = "recursive_character_splitter"
+```
+
+#### 1.3 Embedding Generation
+- **Model**: Google Universal Sentence Encoder (USE)
+- **Vector dimension**: 512
+- **Normalization**: L2 normalization
+- **Storage**: ChromaDB vector database
+
+### 2. Vector Database Architecture
+
+#### 2.1 ChromaDB Configuration
+```python
+# Database settings
+collection_name = f"user_{user_id}_diary"
+metadata = {
+    "user_id": user_id,
+    "source": "diary_entry",
+    "date": entry_date,
+    "chunk_id": chunk_id
+}
+```
+
+#### 2.2 Index Structure
+- **Primary key**: `user_id + chunk_id`
+- **Vector index**: HNSW (Hierarchical Navigable Small World)
+- **Distance metric**: Cosine similarity
+- **Sharding**: Per-user collections
+
+### 3. Retrieval Engine
+
+#### 3.1 Query Processing
+```python
+# Query preprocessing
+def process_query(query: str):
+    # 1. Text cleaning
+    # 2. Stop word removal
+    # 3. Lemmatization
+    # 4. Query expansion
+    return processed_query
+```
+
+#### 3.2 Vector Search
+- **Search algorithm**: K-Nearest Neighbors (KNN)
+- **Top-k results**: 5-10 most relevant chunks
+- **Similarity threshold**: 0.7 (cosine similarity)
+- **Reranking**: Semantic relevance scoring
+
+#### 3.3 Context Assembly
+```python
+# Context building
+def build_context(retrieved_chunks, query):
+    # 1. Sort by relevance score
+    # 2. Remove duplicates
+    # 3. Truncate to token limit
+    # 4. Add metadata context
+    return final_context
+```
+
+### 4. Generation Engine
+
+#### 4.1 LLM Integration
+- **Primary model**: OpenAI GPT-3.5/4
+- **Fallback model**: Local model (nếu cần)
+- **Temperature**: 0.7 (balanced creativity)
+- **Max tokens**: 500 (response length)
+
+#### 4.2 Prompt Engineering
+```python
+# System prompt template
+SYSTEM_PROMPT = """
+You are a helpful AI assistant that answers questions about personal diary entries.
+Use only the provided context to answer questions.
+If the information is not in the context, say so.
+Be conversational but professional.
+"""
+```
+
+#### 4.3 Response Generation
+```python
+# Generation pipeline
+def generate_response(query, context, chat_history):
+    # 1. Build prompt with context
+    # 2. Call LLM API
+    # 3. Post-process response
+    # 4. Validate against context
+    # 5. Return final answer
+```
+
+## 🔧 Cấu hình kỹ thuật
+
+### Performance Tuning
+
+#### 1. Chunking Optimization
+- **Optimal chunk size**: 1000 characters
+- **Overlap ratio**: 20%
+- **Chunking strategy**: Recursive character splitter
+
+#### 2. Vector Search Optimization
+- **Index type**: HNSW
+- **Search parameters**: 
+  - `ef_construction`: 200
+  - `ef_search`: 100
+  - `m`: 16
+
+#### 3. Caching Strategy
+- **Query cache**: Redis (in-memory)
+- **Embedding cache**: Local file cache
+- **Response cache**: TTL-based expiration
+
+### Scalability Features
+
+#### 1. Multi-User Support
+- **User isolation**: Separate vector collections
+- **Resource management**: Per-user memory limits
+- **Concurrent access**: Async processing
+
+#### 2. Horizontal Scaling
+- **Load balancing**: Multiple RAG instances
+- **Database sharding**: User-based distribution
+- **Microservices**: Modular architecture
+
+## 📈 Monitoring & Analytics
+
+### 1. Performance Metrics
+- **Query latency**: < 2 seconds
+- **Retrieval accuracy**: > 85%
+- **Generation quality**: User satisfaction score
+- **System throughput**: Queries per second
+
+### 2. Quality Assurance
+- **Context relevance**: Similarity score tracking
+- **Answer accuracy**: Human evaluation
+- **User feedback**: Rating system
+- **A/B testing**: Model comparison
+
+## 🚀 Deployment Architecture
+
+### 1. Development Environment
+```
+┌─────────────────┐    ┌─────────────────┐
+│   Local Python │    │   Local         │
+│   Environment  │◄──►│   ChromaDB      │
+└─────────────────┘    └─────────────────┘
+```
+
+### 2. Production Environment
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Load Balancer│    │   RAG Service   │    │   Vector DB     │
+│   (Nginx)      │◄──►│   (FastAPI)     │◄──►│   (ChromaDB)    │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                              │
+                              ▼
+                       ┌─────────────────┐
+                       │   Redis Cache   │
+                       └─────────────────┘
+```
+
+## 🔒 Security & Privacy
+
+### 1. Data Protection
+- **User isolation**: Strict separation of data
+- **Encryption**: At-rest and in-transit
+- **Access control**: Role-based permissions
+- **Audit logging**: Complete access history
+
+### 2. Privacy Compliance
+- **GDPR compliance**: Data portability
+- **Data retention**: Configurable policies
+- **User consent**: Explicit permission management
+- **Data anonymization**: Optional features
+
+## 🧪 Testing Strategy
+
+### 1. Unit Testing
+- **Component testing**: Individual modules
+- **Mock testing**: External API simulation
+- **Coverage target**: > 90%
+
+### 2. Integration Testing
+- **End-to-end testing**: Complete RAG pipeline
+- **Performance testing**: Load and stress tests
+- **Security testing**: Vulnerability assessment
+
+
+## 📚 Best Practices
+
+### 1. Model Selection
+- **Embedding models**: Domain-specific fine-tuning
+- **LLM selection**: Cost-performance balance
+- **Fallback strategies**: Graceful degradation
+
+### 2. Data Quality
+- **Input validation**: Strict data checking
+- **Cleaning pipeline**: Automated preprocessing
+- **Quality metrics**: Continuous monitoring
+
+### 3. Error Handling
+- **Graceful failures**: User-friendly error messages
+- **Retry mechanisms**: Automatic recovery
+- **Logging**: Comprehensive error tracking
+
+## 🔮 Future Enhancements
+
+### 1. Advanced Features
+- **Multi-modal RAG**: Image and text processing
+- **Temporal reasoning**: Time-based queries
+- **Emotional analysis**: Sentiment-aware responses
+
+### 2. Performance Improvements
+- **Vector quantization**: Reduced memory usage
+- **Approximate search**: Faster retrieval
+- **Model distillation**: Smaller, faster models
+
+### 3. Integration Capabilities
+- **API ecosystem**: Third-party integrations
+- **Mobile applications**: Native mobile support
+- **Voice interface**: Speech-to-text integration

clean_repo/clean_repo/README.md

@@ -0,0 +1,310 @@
+# RAG Personal Diary Chatbot
+
+## 📖 Project Description
+
+RAG Personal Diary Chatbot is an intelligent chatbot application that uses RAG (Retrieval-Augmented Generation) architecture to interact with users' personal diaries. The application allows users to ask questions about diary content and receive accurate answers based on actual data.
+
+## ✨ Key Features
+
+
+## 🏗️ System Architecture
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   Streamlit UI  │    │   FastAPI       │    │   Vector        │
+│   (Frontend)    │◄──►│   Backend       │◄──►│   Database     │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+                              │
+                              ▼
+                       ┌─────────────────┐
+                       │   RAG Engine    │
+                       │   (LLM +       │
+                       │   Retrieval)    │
+                       └─────────────────┘
+```
+
+## 🚀 Installation and Setup
+
+### System Requirements
+
+### Install Dependencies
+```bash
+# Create virtual environment
+python -m venv .venv
+
+# Activate virtual environment
+# Windows
+.venv\Scripts\activate
+# Linux/Mac
+source .venv/bin/activate
+
+# Install packages
+pip install -r requirements.txt
+```
+
+### Environment Configuration
+
+Create a `.env` file in the project root directory with the following structure:
+
+```env
+# API Keys
+OPENAI_API_KEY=your_openai_api_key_here
+GOOGLE_API_KEY=your_google_api_key_here
+ANTHROPIC_API_KEY=your_anthropic_api_key_here
+
+# Database Configuration
+DATABASE_URL=sqlite:///./user_database/auth.db
+VECTOR_DB_PATH=./VectorDB
+
+# Model Configuration
+EMBEDDING_MODEL=google-universal-sentence-encoder
+LLM_MODEL=gpt-3.5-turbo
+CHUNK_SIZE=1000
+CHUNK_OVERLAP=200
+
+# Server Configuration
+RAG_SERVICE_PORT=8001
+STREAMLIT_PORT=8501
+FASTAPI_PORT=8000
+
+# Security
+SECRET_KEY=your_secret_key_here
+JWT_SECRET=your_jwt_secret_here
+
+# Logging
+LOG_LEVEL=INFO
+LOG_FILE=./logs/app.log
+
+# Vector Database
+CHROMA_DB_PATH=./VectorDB
+PERSIST_DIRECTORY=./VectorDB
+
+# File Processing
+SUPPORTED_FORMATS=pdf,docx,txt,md
+MAX_FILE_SIZE=10485760
+TEMP_DIR=./temp
+
+# RAG Configuration
+TOP_K_RESULTS=5
+SIMILARITY_THRESHOLD=0.7
+MAX_TOKENS=4096
+TEMPERATURE=0.7
+```
+
+**Important Notes:**
+
+### Run the Application
+
+#### 1. Start RAG Service
+```bash
+python start_rag_service.py
+```
+Service will run at: http://127.0.0.1:8001
+
+#### 2. Start Streamlit UI
+```bash
+cd src/streamlit_app
+streamlit run interface.py
+```
+UI will run at: http://localhost:8501
+
+## 📁 Directory Structure
+
+```
+RAG-Personal-Diary-Chatbot/
+├── src/
+│   ├── Indexingstep/          # Data indexing pipeline
+│   ├── Retrivel_And_Generation/  # RAG engine
+│   ├── rag_service/           # FastAPI backend
+│   ├── streamlit_app/         # User interface
+│   └── VectorDB/              # Vector database
+├── notebook/                   # Jupyter notebooks
+├── tests/                      # Unit tests
+├── images/                     # Documentation images
+├── start_rag_service.py       # Service startup script
+├── .env                       # Environment variables (create from template)
+├── env_template.txt           # Environment variables template
+└── README.md
+```
+
+## 🔧 Configuration
+
+### Vector Database
+
+### AI Models
+
+## 📊 Performance
+
+
+## 🧪 Testing
+
+```bash
+# Run all tests
+python -m pytest tests/
+
+# Run specific test
+python -m pytest tests/test_rag_system.py
+```
+
+## 🤝 Contributing
+
+1. Fork the project
+2. Create feature branch (`git checkout -b feature/AmazingFeature`)
+3. Commit changes (`git commit -m 'Add some AmazingFeature'`)
+4. Push to branch (`git push origin feature/AmazingFeature`)
+5. Open Pull Request
+
+## 📝 License
+
+This project is distributed under the MIT License. See the `LICENSE` file for more details.
+
+## 📞 Contact
+
+
+## 🙏 Acknowledgments
+
+
+## 📖 Project Description
+
+RAG Personal Diary Chatbot is an intelligent chatbot application that leverages Retrieval-Augmented Generation (RAG) architecture to interact with users' personal diaries. Users can ask questions about their diary content and receive accurate, context-based answers.
+
+## ✨ Key Features
+
+- **Diary Indexing**: Automatically processes and indexes diary files (PDF, DOCX, TXT)
+- **Semantic Search**: Uses a vector database for semantic search
+- **AI Chatbot**: Natural interaction with diary data
+- **User Isolation**: Each user has a separate vector database
+- **Web Interface**: Easy-to-use Streamlit UI
+- **REST API**: FastAPI backend for integration
+
+## 🏗️ System Architecture
+
+```
+┌───────────────┐    ┌───────────────┐    ┌───────────────┐
+│ Streamlit UI  │◄──►│   FastAPI     │◄──►│  Vector DB    │
+│ (Frontend)    │    │   Backend     │    │  (ChromaDB)   │
+└───────────────┘    └───────────────┘    └───────────────┘
+                              │
+                              ▼
+                       ┌───────────────┐
+                       │   RAG Engine  │
+                       │ (LLM +        │
+                       │  Retrieval)   │
+                       └───────────────┘
+```
+
+## 🚀 Installation and Setup
+
+### System Requirements
+
+- Python 3.8+
+
+### Install Dependencies
+
+```bash
+# Create virtual environment
+python -m venv .venv
+
+# Activate virtual environment
+# Windows
+.venv\Scripts\activate
+# Linux/Mac
+source .venv/bin/activate
+
+# Install packages
+pip install -r requirements.txt
+```
+
+### Environment Configuration
+
+Create a `.env` file in the project root directory with the following structure:
+
+```env
+# Google API Configuration for RAG System
+GOOGLE_API_KEY=[Google API key]
+
+# Database Configuration
+DATABASE_PATH=./src/streamlit_app/backend/diary.db
+
+# Vector Database Configuration
+VECTOR_DB_PATH=./src/Indexingstep/diary_vector_db_enhanced
+COLLECTION_NAME=diary_entries
+
+# RAG Configuration
+EMBEDDING_MODEL=models/embedding-001
+CHAT_MODEL=gemini-2.5-flash
+```
+
+**Important Notes:**
+- Replace all placeholder values with your actual API keys and configuration
+- Keep your `.env` file secure and never commit it to version control
+- The `.env` file is already included in `.gitignore`
+- Use `env_template.txt` as a reference to create your `.env` file
+
+### Run the Application
+
+```bash
+# Start the RAG backend service
+python start_rag_service.py
+
+# Start the Streamlit UI
+streamlit run src/streamlit_app/interface.py
+```
+
+## 📁 Directory Structure
+
+```
+RAG-Personal-Diary-Chatbot/
+├── src/
+│   ├── Indexingstep/          # Data indexing pipeline
+│   ├── Retrivel_And_Generation/  # RAG engine
+│   ├── rag_service/           # FastAPI backend
+│   ├── streamlit_app/         # User interface
+│   └── VectorDB/              # Vector database
+├── notebook/                  # Jupyter notebooks
+├── tests/                     # Unit tests
+├── images/                    # Documentation images
+├── start_rag_service.py       # Service startup script
+├── .env                       # Environment variables (create from template)
+├── env_template.txt           # Environment variables template
+└── README.md
+```
+
+## 🔧 Configuration
+
+### Vector Database
+- **ChromaDB**: Main database for vector embeddings
+- **Chunk size**: 1000 characters (customizable)
+- **Overlap**: 200 characters between chunks
+
+### AI Models
+- **Embedding**: Google's Universal Sentence Encoder
+- **LLM**: Google Gemini (can be replaced with other models)
+
+## 📊 Performance
+
+- **Processing time**: ~2-5 seconds per question
+- **Accuracy**: 85-95% depending on data quality
+- **Scalability**: Supports thousands of diaries
+
+
+## 🤝 Contributing
+
+1. Fork the project
+2. Create a feature branch (`git checkout -b feature/AmazingFeature`)
+3. Commit your changes (`git commit -m 'Add some AmazingFeature'`)
+4. Push to the branch (`git push origin feature/AmazingFeature`)
+5. Open a Pull Request
+## 📞 Contact
+
+- **Author**: [huytrao]
+- **Email**: [traohuy098@gmail.com]
+- **GitHub**: [github.com/huytrao]
+
+## 🙏 Acknowledgments
+
+- Gemini for GPT models
+- Google for Universal Sentence Encoder
+- ChromaDB team for vector database
+- FastAPI and Streamlit communities
+- RAG architecture

clean_repo/clean_repo/app.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""
+Start RAG Service for Personal Diary Chatbot
+"""
+import subprocess
+import sys
+import os
+from pathlib import Path
+import time
+
+def check_requirements():
+    """Check if required packages are installed."""
+    required_packages = ['fastapi', 'uvicorn']
+    missing_packages = []
+    
+    for package in required_packages:
+        try:
+            __import__(package)
+        except ImportError:
+            missing_packages.append(package)
+    
+    if missing_packages:
+        print(f"❌ Missing packages: {', '.join(missing_packages)}")
+        print(f"Install with: pip install {' '.join(missing_packages)}")
+        return False
+    
+    return True
+
+def setup_environment():
+    """Setup environment and directories."""
+    # Ensure VectorDB directory exists
+    vector_db_dir = Path("src/VectorDB")
+    vector_db_dir.mkdir(parents=True, exist_ok=True)
+    print(f"📁 Vector DB directory: {vector_db_dir.absolute()}")
+    
+    # Check for .env file
+    env_file = Path("src/Indexingstep/.env")
+    if env_file.exists():
+        print(f"✅ Environment file found: {env_file}")
+    else:
+        print(f"⚠️  Environment file not found: {env_file}")
+        print("Make sure GOOGLE_API_KEY is set in environment")
+
+def start_service():
+    """Start the RAG FastAPI service."""
+    if not check_requirements():
+        return
+    
+    setup_environment()
+    
+    service_file = Path("src/rag_service/main.py")
+    
+    if not service_file.exists():
+        print(f"❌ Service file not found: {service_file}")
+        print("Please create the RAG service file first")
+        return
+    
+    print("🚀 Starting RAG Service...")
+    print("📍 Service URL: http://0.0.0.0:8001")
+    print("📖 API Docs: http://0.0.0.0:8001/docs")
+    print("💾 Vector databases will be stored in: src/VectorDB/")
+    print("\nPress Ctrl+C to stop the service")
+    print("-" * 50)
+    
+    try:
+        # Change to project root directory
+        os.chdir(Path(__file__).parent)
+        
+        # Start the service in the background
+        process = subprocess.Popen([
+            sys.executable, "-m", "uvicorn",
+            "src.rag_service.main:app",
+            "--host", "0.0.0.0",
+            "--port", "8001",
+            "--reload"
+        ])
+        print(f"🔄 RAG Service running in background (PID: {process.pid})")
+        return process
+    except Exception as e:
+        print(f"❌ Error starting service: {e}")
+        return None
+
+def start_streamlit():
+    # Start Streamlit UI on port 7860 (default for Spaces)
+    os.system("streamlit run src/streamlit_app/interface.py --server.port 7860")
+
+if __name__ == "__main__":
+    start_service()
+    time.sleep(3)
+    start_streamlit()

clean_repo/clean_repo/env_template.txt

@@ -0,0 +1,51 @@
+# Environment Variables Template for RAG Personal Diary Chatbot
+# Copy this file to .env and fill in your actual values
+
+# API Keys
+OPENAI_API_KEY=your_openai_api_key_here
+GOOGLE_API_KEY=your_google_api_key_here
+ANTHROPIC_API_KEY=your_anthropic_api_key_here
+
+# Database Configuration
+DATABASE_URL=sqlite:///./user_database/auth.db
+VECTOR_DB_PATH=./VectorDB
+
+# Model Configuration
+EMBEDDING_MODEL=google-universal-sentence-encoder
+LLM_MODEL=gpt-3.5-turbo
+CHUNK_SIZE=1000
+CHUNK_OVERLAP=200
+
+# Server Configuration
+RAG_SERVICE_PORT=8001
+STREAMLIT_PORT=8501
+FASTAPI_PORT=8000
+
+# Security
+SECRET_KEY=your_secret_key_here
+JWT_SECRET=your_jwt_secret_here
+
+# Logging
+LOG_LEVEL=INFO
+LOG_FILE=./logs/app.log
+
+# Vector Database
+CHROMA_DB_PATH=./VectorDB
+PERSIST_DIRECTORY=./VectorDB
+
+# File Processing
+SUPPORTED_FORMATS=pdf,docx,txt,md
+MAX_FILE_SIZE=10485760
+TEMP_DIR=./temp
+
+# RAG Configuration
+TOP_K_RESULTS=5
+SIMILARITY_THRESHOLD=0.7
+MAX_TOKENS=4096
+TEMPERATURE=0.7
+
+# Instructions:
+# 1. Copy this file to .env
+# 2. Replace all placeholder values with your actual configuration
+# 3. Never commit .env to version control
+# 4. Keep your API keys secure

clean_repo/clean_repo/src/streamlit_app.py

@@ -0,0 +1,40 @@
+import altair as alt
+import numpy as np
+import pandas as pd
+import streamlit as st
+
+"""
+# Welcome to Streamlit!
+
+Edit `/streamlit_app.py` to customize this app to your heart's desire :heart:.
+If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
+forums](https://discuss.streamlit.io).
+
+In the meantime, below is an example of what you can do with just a few lines of code:
+"""
+
+num_points = st.slider("Number of points in spiral", 1, 10000, 1100)
+num_turns = st.slider("Number of turns in spiral", 1, 300, 31)
+
+indices = np.linspace(0, 1, num_points)
+theta = 2 * np.pi * num_turns * indices
+radius = indices
+
+x = radius * np.cos(theta)
+y = radius * np.sin(theta)
+
+df = pd.DataFrame({
+    "x": x,
+    "y": y,
+    "idx": indices,
+    "rand": np.random.randn(num_points),
+})
+
+st.altair_chart(alt.Chart(df, height=700, width=700)
+    .mark_point(filled=True)
+    .encode(
+        x=alt.X("x", axis=None),
+        y=alt.Y("y", axis=None),
+        color=alt.Color("idx", legend=None, scale=alt.Scale()),
+        size=alt.Size("rand", legend=None, scale=alt.Scale(range=[1, 150])),
+    ))

clean_repo/env_template.txt

@@ -0,0 +1,51 @@
+# Environment Variables Template for RAG Personal Diary Chatbot
+# Copy this file to .env and fill in your actual values
+
+# API Keys
+OPENAI_API_KEY=your_openai_api_key_here
+GOOGLE_API_KEY=your_google_api_key_here
+ANTHROPIC_API_KEY=your_anthropic_api_key_here
+
+# Database Configuration
+DATABASE_URL=sqlite:///./user_database/auth.db
+VECTOR_DB_PATH=./VectorDB
+
+# Model Configuration
+EMBEDDING_MODEL=google-universal-sentence-encoder
+LLM_MODEL=gpt-3.5-turbo
+CHUNK_SIZE=1000
+CHUNK_OVERLAP=200
+
+# Server Configuration
+RAG_SERVICE_PORT=8001
+STREAMLIT_PORT=8501
+FASTAPI_PORT=8000
+
+# Security
+SECRET_KEY=your_secret_key_here
+JWT_SECRET=your_jwt_secret_here
+
+# Logging
+LOG_LEVEL=INFO
+LOG_FILE=./logs/app.log
+
+# Vector Database
+CHROMA_DB_PATH=./VectorDB
+PERSIST_DIRECTORY=./VectorDB
+
+# File Processing
+SUPPORTED_FORMATS=pdf,docx,txt,md
+MAX_FILE_SIZE=10485760
+TEMP_DIR=./temp
+
+# RAG Configuration
+TOP_K_RESULTS=5
+SIMILARITY_THRESHOLD=0.7
+MAX_TOKENS=4096
+TEMPERATURE=0.7
+
+# Instructions:
+# 1. Copy this file to .env
+# 2. Replace all placeholder values with your actual configuration
+# 3. Never commit .env to version control
+# 4. Keep your API keys secure

clean_repo/src/Indexingstep/Datasplitting.py

@@ -0,0 +1,44 @@
+from langchain_text_splitters import CharacterTextSplitter
+
+class DataSplitting:
+    def __init__(self, chunk_size=1000, chunk_overlap=200, separator="\n\n"):
+        """
+        Initialize the DataSplitting class.
+        
+        Args:
+            chunk_size (int): Maximum size of each chunk
+            chunk_overlap (int): Number of characters to overlap between chunks
+            separator (str): Character(s) to split on
+        """
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.separator = separator
+        self.text_splitter = CharacterTextSplitter(
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+            separator=self.separator
+        )
+    
+    def split_text(self, text):
+        """
+        Split the input text into chunks.
+        
+        Args:
+            text (str): The text to be split
+            
+        Returns:
+            list: List of text chunks
+        """
+        return self.text_splitter.split_text(text)
+    
+    def split_documents(self, documents):
+        """
+        Split documents into chunks.
+        
+        Args:
+            documents (list): List of documents to be split
+            
+        Returns:
+            list: List of document chunks
+        """
+        return self.text_splitter.split_documents(documents)

clean_repo/src/Indexingstep/database_utils.py

@@ -0,0 +1,140 @@
+"""
+Database utilities and context managers.
+"""
+
+import sqlite3
+import os
+from contextlib import contextmanager
+from typing import Generator
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+@contextmanager
+def open_db(db_path: str) -> Generator[sqlite3.Connection, None, None]:
+    """
+    Context manager for database connections.
+    
+    Args:
+        db_path: Path to the SQLite database
+        
+    Yields:
+        Database connection
+    """
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        conn.row_factory = sqlite3.Row
+        yield conn
+    except Exception as e:
+        if conn:
+            conn.rollback()
+        logger.error(f"Database error with {db_path}: {e}")
+        raise
+    finally:
+        if conn:
+            conn.close()
+
+
+def ensure_database_exists(db_path: str, user_id: int) -> None:
+    """
+    Ensure user-specific database exists with proper schema.
+    
+    Args:
+        db_path: Path to the database file
+        user_id: User ID for default value
+    """
+    if os.path.exists(db_path):
+        return
+    
+    # Create directory if it doesn't exist
+    os.makedirs(os.path.dirname(db_path), exist_ok=True)
+    
+    with open_db(db_path) as conn:
+        cursor = conn.cursor()
+        
+        # Create table schema
+        cursor.execute(f"""
+            CREATE TABLE IF NOT EXISTS diary_entries (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                user_id INTEGER NOT NULL DEFAULT {user_id},
+                date TEXT NOT NULL,
+                content TEXT NOT NULL,
+                tags TEXT DEFAULT '',
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+        
+        # Create index
+        cursor.execute("""
+            CREATE INDEX IF NOT EXISTS idx_user_date ON diary_entries(user_id, date)
+        """)
+        
+        conn.commit()
+        
+    logger.info(f"Created user database: {db_path}")
+
+
+def migrate_user_data(source_db_path: str, target_db_path: str, user_id: int) -> int:
+    """
+    Migrate user data from shared database to user-specific database.
+    
+    Args:
+        source_db_path: Path to source database
+        target_db_path: Path to target database
+        user_id: User ID to migrate
+        
+    Returns:
+        Number of entries migrated
+    """
+    if not os.path.exists(source_db_path):
+        return 0
+    
+    migrated_count = 0
+    
+    try:
+        with open_db(source_db_path) as source_conn:
+            with open_db(target_db_path) as target_conn:
+                source_cursor = source_conn.cursor()
+                target_cursor = target_conn.cursor()
+                
+                # Check if shared DB has user_id column
+                source_cursor.execute("PRAGMA table_info(diary_entries)")
+                columns = [col[1] for col in source_cursor.fetchall()]
+                
+                if 'user_id' in columns:
+                    # Migrate specific user data
+                    source_cursor.execute("""
+                        SELECT date, content, tags, created_at 
+                        FROM diary_entries 
+                        WHERE user_id = ?
+                    """, (user_id,))
+                else:
+                    # If no user_id column, migrate all data to user 1 only
+                    if user_id == 1:
+                        source_cursor.execute("""
+                            SELECT date, content, COALESCE(tags, ''), created_at 
+                            FROM diary_entries
+                        """)
+                    else:
+                        return 0
+                
+                rows = source_cursor.fetchall()
+                
+                for row in rows:
+                    target_cursor.execute("""
+                        INSERT OR IGNORE INTO diary_entries (user_id, date, content, tags, created_at)
+                        VALUES (?, ?, ?, ?, ?)
+                    """, (user_id, row[0], row[1], row[2] if len(row) > 2 else '', row[3] if len(row) > 3 else None))
+                
+                target_conn.commit()
+                migrated_count = len(rows)
+        
+        if migrated_count > 0:
+            logger.info(f"Migrated {migrated_count} entries for user {user_id}")
+    
+    except Exception as e:
+        logger.warning(f"Could not migrate data for user {user_id}: {e}")
+    
+    return migrated_count

clean_repo/src/Indexingstep/dataloading.py

@@ -0,0 +1,603 @@
+import sqlite3
+from typing import List, Optional, Dict, Any
+from langchain.schema import Document
+from langchain.document_loaders.base import BaseLoader
+import logging
+import re
+from datetime import datetime
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class DiaryDataLoader(BaseLoader):
+    """
+    Custom LangChain document loader for diary entries from SQLite database.
+    Enhanced with detailed metadata extraction for better indexing.
+    """
+    
+    def __init__(
+        self, 
+        db_path: str,
+        table_name: str = "diary_entries",
+        content_column: str = "content",
+        date_column: str = "date",
+        tags_column: str = "tags",
+        id_column: str = "id",
+        user_id: int = 1
+    ):
+        """
+        Initialize the DiaryDataLoader.
+        
+        Args:
+            db_path (str): Path to the SQLite database file
+            table_name (str): Name of the table containing diary entries
+            content_column (str): Name of the column containing diary content
+            date_column (str): Name of the column containing entry dates
+            tags_column (str): Name of the column containing entry tags
+            id_column (str): Name of the column containing entry IDs
+            user_id (int): ID of the user for filtering diary entries
+        """
+        self.db_path = db_path
+        self.table_name = table_name
+        self.content_column = content_column
+        self.date_column = date_column
+        self.tags_column = tags_column
+        self.id_column = id_column
+        self.user_id = user_id
+    
+    def _extract_tags_from_content(self, content: str) -> List[str]:
+        """
+        Extract #tags from content string.
+        
+        Args:
+            content: The diary content string
+            
+        Returns:
+            List of tags found (without # symbol)
+        """
+        if not content:
+            return []
+        
+        # Find all #tags in content
+        tag_pattern = r'#(\w+(?:[_-]\w+)*)'
+        matches = re.findall(tag_pattern, content, re.IGNORECASE)
+        
+        # Remove duplicates and return lowercase tags
+        return list(set([tag.lower() for tag in matches]))
+    
+    def _extract_location_from_content(self, content: str) -> Optional[str]:
+        """
+        Extract location information from content using common patterns.
+        
+        Args:
+            content: The diary content string
+            
+        Returns:
+            Location string if found, None otherwise
+        """
+        if not content:
+            return None
+        
+        # Common location patterns
+        location_patterns = [
+            r'at\s+([A-Z][a-zA-Z\s]+(?:Park|Beach|Mall|Store|Restaurant|Cafe|Office|Home|School|University))',
+            r'in\s+([A-Z][a-zA-Z\s]+(?:City|District|Area|Street|Road))',
+            r'went\s+to\s+([A-Z][a-zA-Z\s]+)',
+            r'visited\s+([A-Z][a-zA-Z\s]+)',
+            r'location:\s*([A-Za-z\s]+)',
+            r'place:\s*([A-Za-z\s]+)'
+        ]
+        
+        for pattern in location_patterns:
+            matches = re.findall(pattern, content, re.IGNORECASE)
+            if matches:
+                return matches[0].strip()
+        
+        return None
+    
+    def _extract_people_from_content(self, content: str) -> List[str]:
+        """
+        Extract people/relationships mentioned in content.
+        
+        Args:
+            content: The diary content string
+            
+        Returns:
+            List of people/relationships mentioned
+        """
+        if not content:
+            return []
+        
+        # Common relationship patterns
+        people_patterns = [
+            r'with\s+(my\s+)?(\w+(?:\s+\w+)?)',
+            r'(mom|dad|mother|father|sister|brother|friend|colleague|boss|teacher)',
+            r'(family|friends|team|colleagues)',
+            r'met\s+([\w\s]+)',
+            r'talked\s+to\s+([\w\s]+)'
+        ]
+        
+        people = set()
+        for pattern in people_patterns:
+            matches = re.findall(pattern, content, re.IGNORECASE)
+            for match in matches:
+                if isinstance(match, tuple):
+                    for part in match:
+                        if part.strip():
+                            people.add(part.strip().lower())
+                else:
+                    people.add(match.strip().lower())
+        
+        # Filter out common words that are not people
+        exclude_words = {'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by'}
+        people = [p for p in people if p not in exclude_words and len(p) > 2]
+        
+        return list(people)
+    
+    def _get_day_of_week(self, date_str: str) -> str:
+        """
+        Get day of week from date string.
+        
+        Args:
+            date_str: Date string in YYYY-MM-DD format
+            
+        Returns:
+            Day of week (e.g., 'Monday', 'Tuesday', etc.)
+        """
+        try:
+            date_obj = datetime.strptime(date_str, '%Y-%m-%d')
+            return date_obj.strftime('%A')
+        except:
+            return 'Unknown'
+    
+    def _extract_content_from_structured_format(self, raw_content: str) -> tuple:
+        """
+        Extract actual content from structured format like:
+        Title: xxxx
+        Type: Text
+        Content: actual content here
+        
+        Returns:
+            tuple: (title, actual_content)
+        """
+        lines = raw_content.strip().split('\n')
+        title = ""
+        content = ""
+        
+        for line in lines:
+            if line.startswith("Title: "):
+                title = line.replace("Title: ", "").strip()
+            elif line.startswith("Content: "):
+                content = line.replace("Content: ", "").strip()
+        
+        # If no structured format found, return original content
+        if not content:
+            content = raw_content
+            
+        return title, content
+    
+    def load(self) -> List[Document]:
+        """
+        Load diary entries from the database and convert them to LangChain Documents.
+        
+        Returns:
+            List[Document]: List of LangChain Document objects
+        """
+        documents = []
+        
+        try:
+            # Connect to the SQLite database
+            conn = sqlite3.connect(self.db_path)
+            conn.row_factory = sqlite3.Row  # Enable accessing columns by name
+            cursor = conn.cursor()
+            
+            # Build the SQL query with all required columns
+            columns = [self.id_column, self.date_column, self.content_column, self.tags_column]
+            
+            query = f"SELECT {', '.join(columns)} FROM {self.table_name} WHERE user_id = ? ORDER BY {self.date_column} DESC"
+            
+            # Execute the query
+            cursor.execute(query, (self.user_id,))
+            rows = cursor.fetchall()
+            
+            logger.info(f"Loaded {len(rows)} diary entries from database")
+            
+            # Convert each row to a LangChain Document with enhanced metadata
+            for row in rows:
+                row_dict = dict(row) if hasattr(row, 'keys') else {
+                    self.id_column: row[0],
+                    self.date_column: row[1], 
+                    self.content_column: row[2],
+                    self.tags_column: row[3] if len(row) > 3 else ""
+                }
+                
+                raw_content = row_dict[self.content_column]
+                date = row_dict[self.date_column]
+                entry_id = row_dict.get(self.id_column, "unknown")
+                db_tags = row_dict.get(self.tags_column, "")
+                
+                # Extract structured content
+                title, actual_content = self._extract_content_from_structured_format(raw_content)
+                
+                # Extract comprehensive metadata
+                content_tags = self._extract_tags_from_content(actual_content)
+                db_tag_list = [tag.strip() for tag in db_tags.split(',') if tag.strip()] if db_tags else []
+                all_tags = list(set(content_tags + db_tag_list))  # Combine and deduplicate
+                
+                location = self._extract_location_from_content(actual_content)
+                people = self._extract_people_from_content(actual_content)
+                day_of_week = self._get_day_of_week(date)
+                
+                # Create comprehensive metadata for the document
+                metadata = {
+                    "source": self.db_path,
+                    "entry_id": str(entry_id),
+                    "date": date,
+                    "day_of_week": day_of_week,
+                    "type": "diary_entry",
+                    "tags": all_tags,
+                    "tag_count": len(all_tags),
+                    "content_length": len(actual_content),
+                    "word_count": len(actual_content.split())
+                }
+                
+                # Add optional metadata if available
+                if title:
+                    metadata["title"] = title
+                if location:
+                    metadata["location"] = location
+                if people:
+                    metadata["people"] = people
+                    metadata["people_count"] = len(people)
+                
+                # Add mood/sentiment tags if present
+                mood_tags = [tag for tag in all_tags if tag in ['happy', 'sad', 'excited', 'tired', 'angry', 'peaceful', 'stressed', 'grateful', 'frustrated', 'motivated']]
+                if mood_tags:
+                    metadata["mood_tags"] = mood_tags
+                
+                # Create Document object with actual content
+                document = Document(
+                    page_content=actual_content,
+                    metadata=metadata
+                )
+                
+                documents.append(document)
+            
+            conn.close()
+            logger.info(f"Successfully converted {len(documents)} entries to Documents")
+            
+        except sqlite3.Error as e:
+            logger.error(f"Database error: {e}")
+            raise
+        except Exception as e:
+            logger.error(f"Error loading diary data: {e}")
+            raise
+        
+        return documents
+    
+    def load_by_date_range(self, start_date: str, end_date: str) -> List[Document]:
+        """
+        Load diary entries within a specific date range.
+        
+        Args:
+            start_date (str): Start date in YYYY-MM-DD format
+            end_date (str): End date in YYYY-MM-DD format
+            
+        Returns:
+            List[Document]: Filtered list of Document objects
+        """
+        documents = []
+        
+        try:
+            conn = sqlite3.connect(self.db_path)
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+            
+            columns = [self.content_column, self.date_column]
+            # if self.title_column:
+            #     columns.append(self.title_column)
+            
+            query = f"""
+                SELECT {', '.join(columns)} 
+                FROM {self.table_name} 
+                WHERE user_id = ? AND {self.date_column} BETWEEN ? AND ?
+                ORDER BY {self.date_column}
+            """
+            
+            cursor.execute(query, (self.user_id, start_date, end_date))
+            rows = cursor.fetchall()
+            
+            logger.info(f"Loaded {len(rows)} diary entries from {start_date} to {end_date}")
+            
+            for row in rows:
+                raw_content = row[self.content_column]
+                date = row[self.date_column]
+                
+                # Extract structured content
+                title, actual_content = self._extract_content_from_structured_format(raw_content)
+                
+                metadata = {
+                    "source": self.db_path,
+                    "date": date,
+                    "type": "diary_entry",
+                    "date_range": f"{start_date}_to_{end_date}"
+                }
+                
+                # Add title to metadata if available
+                if title:
+                    metadata["title"] = title
+                
+                document = Document(
+                    page_content=actual_content,
+                    metadata=metadata
+                )
+                
+                documents.append(document)
+            
+            conn.close()
+            
+        except sqlite3.Error as e:
+            logger.error(f"Database error: {e}")
+            raise
+        except Exception as e:
+            logger.error(f"Error loading diary data by date range: {e}")
+            raise
+        
+        return documents
+    
+    def get_table_info(self) -> dict:
+        """
+        Get information about the database table structure.
+        
+        Returns:
+            dict: Table information including columns and row count
+        """
+        try:
+            conn = sqlite3.connect(self.db_path)
+            cursor = conn.cursor()
+            
+            # Get table schema
+            cursor.execute(f"PRAGMA table_info({self.table_name})")
+            columns = cursor.fetchall()
+            
+            # Get row count
+            cursor.execute(f"SELECT COUNT(*) FROM {self.table_name}")
+            row_count = cursor.fetchone()[0]
+            
+            conn.close()
+            
+            return {
+                "table_name": self.table_name,
+                "columns": [{"name": col[1], "type": col[2]} for col in columns],
+                "row_count": row_count
+            }
+            
+        except sqlite3.Error as e:
+            logger.error(f"Database error: {e}")
+            raise
+
+class DiaryContentPreprocessor:
+    """
+    Preprocessor for diary content to clean and standardize text before indexing.
+    """
+    
+    def __init__(
+        self,
+        remove_extra_whitespace: bool = True,
+        normalize_line_breaks: bool = True,
+        min_content_length: int = 10,
+        max_content_length: Optional[int] = None
+    ):
+        """
+        Initialize the content preprocessor.
+        
+        Args:
+            remove_extra_whitespace (bool): Remove extra spaces and tabs
+            normalize_line_breaks (bool): Normalize line breaks to single newlines
+            min_content_length (int): Minimum content length to keep
+            max_content_length (int, optional): Maximum content length to keep
+        """
+        self.remove_extra_whitespace = remove_extra_whitespace
+        self.normalize_line_breaks = normalize_line_breaks
+        self.min_content_length = min_content_length
+        self.max_content_length = max_content_length
+    
+    def preprocess_content(self, content: str) -> str:
+        """
+        Preprocess diary content text.
+        
+        Args:
+            content (str): Raw diary content
+            
+        Returns:
+            str: Preprocessed content
+        """
+        if not content or not isinstance(content, str):
+            return ""
+        
+        processed_content = content
+        
+        # Remove extra whitespace
+        if self.remove_extra_whitespace:
+            processed_content = ' '.join(processed_content.split())
+        
+        # Normalize line breaks
+        if self.normalize_line_breaks:
+            processed_content = processed_content.replace('\r\n', '\n').replace('\r', '\n')
+            # Remove multiple consecutive newlines
+            processed_content = re.sub(r'\n+', '\n', processed_content)
+        
+        # Strip leading/trailing whitespace
+        processed_content = processed_content.strip()
+        
+        # Check length constraints
+        if len(processed_content) < self.min_content_length:
+            logger.warning(f"Content too short ({len(processed_content)} chars), skipping")
+            return ""
+        
+        if self.max_content_length and len(processed_content) > self.max_content_length:
+            logger.warning(f"Content too long ({len(processed_content)} chars), truncating")
+            processed_content = processed_content[:self.max_content_length]
+        
+        return processed_content
+    
+    def preprocess_documents(self, documents: List[Document]) -> List[Document]:
+        """
+        Preprocess a list of Document objects.
+        
+        Args:
+            documents (List[Document]): List of documents to preprocess
+            
+        Returns:
+            List[Document]: List of preprocessed documents
+        """
+        preprocessed_docs = []
+        
+        for doc in documents:
+            processed_content = self.preprocess_content(doc.page_content)
+            
+            # Skip empty content after preprocessing
+            if not processed_content:
+                continue
+            
+            # Create new document with processed content
+            preprocessed_doc = Document(
+                page_content=processed_content,
+                metadata=doc.metadata.copy()
+            )
+            
+            preprocessed_docs.append(preprocessed_doc)
+        
+        logger.info(f"Preprocessed {len(documents)} documents, kept {len(preprocessed_docs)}")
+        return preprocessed_docs
+    
+    def load_all_entries(self, user_id: int = None) -> List[Dict[str, Any]]:
+        """
+        Load all diary entries for a specific user.
+        
+        Args:
+            user_id: User ID to filter entries
+            
+        Returns:
+            List of diary entry dictionaries
+        """
+        if user_id is None:
+            user_id = self.user_id
+            
+        entries = []
+        
+        try:
+            conn = sqlite3.connect(self.db_path)
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+            
+            query = f"""
+                SELECT id, user_id, date, content, tags, created_at 
+                FROM {self.table_name} 
+                WHERE user_id = ? 
+                ORDER BY date DESC, created_at DESC
+            """
+            
+            cursor.execute(query, (user_id,))
+            rows = cursor.fetchall()
+            
+            for row in rows:
+                entries.append({
+                    'id': row['id'],
+                    'user_id': row['user_id'],
+                    'date': row['date'],
+                    'content': row['content'],
+                    'tags': row['tags'] or '',
+                    'created_at': row['created_at']
+                })
+            
+            conn.close()
+            logger.info(f"Loaded {len(entries)} entries for user {user_id}")
+            
+        except sqlite3.Error as e:
+            logger.error(f"Database error loading entries: {e}")
+            
+        return entries
+    
+    def load_entries_since(self, since_date, user_id: int = None) -> List[Dict[str, Any]]:
+        """
+        Load diary entries since a specific date.
+        
+        Args:
+            since_date: datetime object or ISO string
+            user_id: User ID to filter entries
+            
+        Returns:
+            List of diary entry dictionaries
+        """
+        if user_id is None:
+            user_id = self.user_id
+            
+        entries = []
+        
+        try:
+            # Convert datetime to string if needed
+            if hasattr(since_date, 'isoformat'):
+                since_str = since_date.isoformat()
+            else:
+                since_str = str(since_date)
+                
+            conn = sqlite3.connect(self.db_path)
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+            
+            query = f"""
+                SELECT id, user_id, date, content, tags, created_at 
+                FROM {self.table_name} 
+                WHERE user_id = ? AND created_at > ?
+                ORDER BY date DESC, created_at DESC
+            """
+            
+            cursor.execute(query, (user_id, since_str))
+            rows = cursor.fetchall()
+            
+            for row in rows:
+                entries.append({
+                    'id': row['id'],
+                    'user_id': row['user_id'],
+                    'date': row['date'],
+                    'content': row['content'],
+                    'tags': row['tags'] or '',
+                    'created_at': row['created_at']
+                })
+            
+            conn.close()
+            logger.info(f"Loaded {len(entries)} entries since {since_str} for user {user_id}")
+            
+        except sqlite3.Error as e:
+            logger.error(f"Database error loading entries since {since_date}: {e}")
+            
+        return entries
+
+# Example usage
+if __name__ == "__main__":
+    # Initialize the loader
+    loader = DiaryDataLoader(
+        db_path="../streamlit_app/backend/diary.db",
+        table_name="diary_entries",
+        content_column="content",
+        date_column="date" #,
+        # title_column="title"  
+    )
+    
+    # Load all documents
+    documents = loader.load()
+    print(f"Loaded {len(documents)} diary entries")
+    
+    # Load documents by date range
+    filtered_docs = loader.load_by_date_range("2024-01-01", "2026-12-31")
+    print(f"Loaded {len(filtered_docs)} entries from 2024")
+    
+    # Get table information
+    table_info = loader.get_table_info()
+    print(f"Table info: {table_info}")
+
+    # view document contents
+    for doc in documents:
+        print(f"Document content: {doc.page_content}")

clean_repo/src/Indexingstep/diary_text_splitter.py

@@ -0,0 +1,241 @@
+"""
+Custom text splitter optimized for diary entries.
+Handles entry-based chunking with smart splitting for long entries.
+"""
+
+from typing import List, Optional, Any, Dict
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.schema import Document
+import logging
+
+logger = logging.getLogger(__name__)
+
+class DiaryTextSplitter:
+    """
+    Custom text splitter optimized for diary entries.
+    
+    Strategy:
+    1. Each diary entry = 1 chunk (for short entries)
+    2. Long entries → split into 200-300 tokens with 50-token sliding window
+    3. Preserve metadata across all chunks
+    """
+    
+    def __init__(
+        self,
+        chunk_size: int = 300,  # ~200-300 tokens
+        chunk_overlap: int = 50,  # ~50 tokens overlap
+        length_function: callable = len,
+        separators: Optional[List[str]] = None
+    ):
+        """
+        Initialize the DiaryTextSplitter.
+        
+        Args:
+            chunk_size: Maximum chunk size in characters (~300 chars ≈ 200-300 tokens)
+            chunk_overlap: Overlap between chunks to preserve context
+            length_function: Function to calculate text length
+            separators: List of separators for splitting (sentence-aware)
+        """
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.length_function = length_function
+        
+        # Diary-optimized separators (sentence and paragraph aware)
+        self.separators = separators or [
+            "\n\n",  # Paragraph breaks
+            "\n",    # Line breaks
+            ". ",    # Sentence endings
+            "! ",    # Exclamation sentences
+            "? ",    # Question sentences
+            "; ",    # Semicolon breaks
+            ", ",    # Comma breaks
+            " ",     # Word breaks
+            ""       # Character breaks (last resort)
+        ]
+        
+        # Initialize recursive character splitter for long entries
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+            length_function=self.length_function,
+            separators=self.separators
+        )
+    
+    def _estimate_tokens(self, text: str) -> int:
+        """
+        Estimate token count from character count.
+        Rule of thumb: ~4 characters per token for English text.
+        
+        Args:
+            text: Input text
+            
+        Returns:
+            Estimated token count
+        """
+        return len(text) // 4
+    
+    def _should_split_entry(self, content: str) -> bool:
+        """
+        Determine if a diary entry should be split into multiple chunks.
+        
+        Args:
+            content: Diary entry content
+            
+        Returns:
+            True if entry should be split, False otherwise
+        """
+        estimated_tokens = self._estimate_tokens(content)
+        # Split if entry is longer than ~250 tokens (considering our 200-300 target)
+        return estimated_tokens > 250
+    
+    def _create_chunk_metadata(self, original_doc: Document, chunk_index: int, total_chunks: int) -> Dict[str, Any]:
+        """
+        Create metadata for a chunk, preserving original metadata.
+        
+        Args:
+            original_doc: Original document
+            chunk_index: Index of current chunk (0-based)
+            total_chunks: Total number of chunks for this entry
+            
+        Returns:
+            Metadata dictionary for the chunk
+        """
+        chunk_metadata = original_doc.metadata.copy()
+        
+        # Add chunk-specific metadata
+        chunk_metadata.update({
+            "chunk_index": chunk_index,
+            "total_chunks": total_chunks,
+            "is_chunked": total_chunks > 1,
+            "chunk_id": f"{chunk_metadata.get('entry_id', 'unknown')}_{chunk_index}"
+        })
+        
+        return chunk_metadata
+    
+    def split_documents(self, documents: List[Document]) -> List[Document]:
+        """
+        Split diary documents into optimized chunks.
+        
+        Args:
+            documents: List of diary entry documents
+            
+        Returns:
+            List of chunked documents with preserved metadata
+        """
+        chunked_documents = []
+        
+        for doc in documents:
+            content = doc.page_content
+            
+            # Check if entry needs splitting
+            if not self._should_split_entry(content):
+                # Keep as single chunk for short entries
+                chunk_metadata = self._create_chunk_metadata(doc, 0, 1)
+                
+                chunked_doc = Document(
+                    page_content=content,
+                    metadata=chunk_metadata
+                )
+                chunked_documents.append(chunked_doc)
+                
+                logger.debug(f"Entry {doc.metadata.get('entry_id', 'unknown')} kept as single chunk")
+                
+            else:
+                # Split long entry into multiple chunks
+                text_chunks = self.text_splitter.split_text(content)
+                total_chunks = len(text_chunks)
+                
+                logger.info(f"Entry {doc.metadata.get('entry_id', 'unknown')} split into {total_chunks} chunks")
+                
+                for i, chunk_text in enumerate(text_chunks):
+                    chunk_metadata = self._create_chunk_metadata(doc, i, total_chunks)
+                    
+                    # Add chunk position information
+                    chunk_metadata["chunk_position"] = "start" if i == 0 else "end" if i == total_chunks - 1 else "middle"
+                    
+                    chunked_doc = Document(
+                        page_content=chunk_text,
+                        metadata=chunk_metadata
+                    )
+                    chunked_documents.append(chunked_doc)
+        
+        logger.info(f"Split {len(documents)} entries into {len(chunked_documents)} chunks")
+        return chunked_documents
+    
+    def get_chunk_stats(self, documents: List[Document]) -> Dict[str, Any]:
+        """
+        Get statistics about chunking results.
+        
+        Args:
+            documents: List of chunked documents
+            
+        Returns:
+            Dictionary with chunking statistics
+        """
+        total_chunks = len(documents)
+        single_chunks = sum(1 for doc in documents if doc.metadata.get("total_chunks", 1) == 1)
+        multi_chunks = total_chunks - single_chunks
+        
+        unique_entries = len(set(doc.metadata.get("entry_id", "unknown") for doc in documents))
+        
+        avg_chunk_size = sum(len(doc.page_content) for doc in documents) / total_chunks if total_chunks > 0 else 0
+        avg_tokens = sum(self._estimate_tokens(doc.page_content) for doc in documents) / total_chunks if total_chunks > 0 else 0
+        
+        return {
+            "total_chunks": total_chunks,
+            "unique_entries": unique_entries,
+            "single_chunk_entries": single_chunks,
+            "multi_chunk_entries": multi_chunks,
+            "avg_chunk_size_chars": round(avg_chunk_size, 2),
+            "avg_chunk_size_tokens": round(avg_tokens, 2),
+            "chunking_ratio": round(total_chunks / unique_entries, 2) if unique_entries > 0 else 0
+        }
+    
+    def split_diary_entry(self, entry: Dict[str, Any]) -> List[Document]:
+        """
+        Split a single diary entry into document chunks.
+        
+        Args:
+            entry: Dictionary containing diary entry data
+            
+        Returns:
+            List of Document objects
+        """
+        # Create Document from entry
+        content = entry.get('content', '')
+        
+        # Extract title from content if it's in structured format
+        title = ""
+        actual_content = content
+        
+        if content.startswith("Title: "):
+            lines = content.split('\n')
+            for line in lines:
+                if line.startswith("Title: "):
+                    title = line.replace("Title: ", "").strip()
+                elif line.startswith("Content: "):
+                    actual_content = line.replace("Content: ", "").strip()
+        
+        # Create metadata
+        metadata = {
+            "entry_id": str(entry.get('id', 'unknown')),
+            "user_id": entry.get('user_id', 1),
+            "date": entry.get('date', ''),
+            "tags": entry.get('tags', ''),
+            "created_at": entry.get('created_at', ''),
+            "type": "diary_entry",
+            "content_length": len(actual_content),
+            "word_count": len(actual_content.split())
+        }
+        
+        if title:
+            metadata["title"] = title
+        
+        # Create Document
+        doc = Document(
+            page_content=actual_content,
+            metadata=metadata
+        )
+        
+        # Split using the existing split_documents method
+        return self.split_documents([doc])

clean_repo/src/Indexingstep/embedding_and_storing.py

@@ -0,0 +1,499 @@
+from langchain_chroma import Chroma
+from langchain.schema import Document
+from typing import List, Optional, Dict, Any, Union
+import os
+import logging
+from pathlib import Path
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class DiaryEmbeddingAndStorage:
+    """
+    Class for embedding diary documents and storing them in Chroma vector database.
+    Enhanced with metadata filtering for ChromaDB compatibility.
+    """
+    
+    def _filter_metadata(self, metadata: Dict[str, Any]) -> Dict[str, Union[str, int, float, bool]]:
+        """
+        Filter metadata to only include types supported by ChromaDB.
+        
+        Args:
+            metadata: Original metadata dictionary
+            
+        Returns:
+            Filtered metadata with only supported types
+        """
+        filtered = {}
+        
+        for key, value in metadata.items():
+            if isinstance(value, (str, int, float, bool)) or value is None:
+                filtered[key] = value
+            elif isinstance(value, list):
+                # Convert lists to comma-separated strings
+                if value:  # Only if list is not empty
+                    filtered[f"{key}_list"] = ", ".join(str(item) for item in value)
+                    filtered[f"{key}_count"] = len(value)
+            elif isinstance(value, dict):
+                # Skip complex nested objects
+                logger.debug(f"Skipping complex metadata field: {key}")
+                continue
+            else:
+                # Convert other types to string
+                filtered[key] = str(value)
+        
+        return filtered
+    
+    def __init__(
+        self,
+        user_id: int = 1,
+        api_key: Optional[str] = None,
+        base_persist_directory: str = "./",
+        embedding_model: str = "models/embedding-001",
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200
+    ):
+        """
+        Initialize the embedding and storage system with user-specific database.
+        
+        Args:
+            user_id (int): User ID for user-specific vector database
+            api_key (str, optional): Google API key for embeddings
+            base_persist_directory (str): Base directory for vector databases
+            embedding_model (str): Google embedding model to use
+            chunk_size (int): Size of text chunks for embedding
+            chunk_overlap (int): Overlap between chunks
+        """
+        # Set up Google API key
+        if api_key:
+            os.environ["GOOGLE_API_KEY"] = api_key
+        elif "GOOGLE_API_KEY" not in os.environ:
+            raise ValueError("Google API key must be provided either as parameter or environment variable")
+        
+        self.user_id = user_id
+        self.base_persist_directory = base_persist_directory
+        
+        # Create user-specific paths
+        self.persist_directory = os.path.join(base_persist_directory, f"user_{user_id}_vector_db")
+        self.collection_name = f"user_{user_id}_diary_entries"
+        
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        
+        # Initialize embedding model
+        try:
+            self.embeddings = GoogleGenerativeAIEmbeddings(
+                model=embedding_model
+            )
+            # logger.info(f"Initialized Google embeddings with model: {embedding_model}")
+        except Exception as e:
+            logger.error(f"Failed to initialize embeddings: {e}")
+            raise
+        
+        # Initialize or load existing vector store
+        self.vector_store = None
+        self._setup_vector_store()
+    
+    def _setup_vector_store(self):
+        """Set up the Chroma vector store."""
+        try:
+            # Create persist directory if it doesn't exist
+            Path(self.persist_directory).mkdir(parents=True, exist_ok=True)
+            
+            # Initialize Chroma vector store
+            self.vector_store = Chroma(
+                collection_name=self.collection_name,
+                embedding_function=self.embeddings,
+                persist_directory=self.persist_directory
+            )
+            
+            # logger.info(f"Vector store initialized with persist directory: {self.persist_directory}")
+            
+        except Exception as e:
+            logger.error(f"Failed to setup vector store: {e}")
+            raise
+    
+    def embed_and_store_documents(self, documents: List[Document]) -> List[str]:
+        """
+        Embed and store documents in the vector database.
+        
+        Args:
+            documents (List[Document]): List of LangChain Document objects
+            
+        Returns:
+            List[str]: List of document IDs
+        """
+        if not documents:
+            logger.warning("No documents provided for embedding")
+            return []
+        
+        try:
+            # Filter metadata for each document
+            filtered_documents = []
+            for doc in documents:
+                filtered_metadata = self._filter_metadata(doc.metadata)
+                filtered_doc = Document(
+                    page_content=doc.page_content,
+                    metadata=filtered_metadata
+                )
+                filtered_documents.append(filtered_doc)
+                
+                # Log metadata transformation for debugging
+                logger.debug(f"Original metadata keys: {list(doc.metadata.keys())}")
+                logger.debug(f"Filtered metadata keys: {list(filtered_metadata.keys())}")
+            
+            # Add documents to vector store
+            document_ids = self.vector_store.add_documents(filtered_documents)
+            
+            # Persist the vector store (auto-persisted in new langchain-chroma)
+            # self.vector_store.persist()  # Not needed in langchain-chroma 0.2+
+            
+            logger.info(f"Successfully embedded and stored {len(filtered_documents)} documents")
+            return document_ids
+            
+        except Exception as e:
+            logger.error(f"Failed to embed and store documents: {e}")
+            raise
+    
+    def embed_and_store_texts(
+        self, 
+        texts: List[str], 
+        metadatas: Optional[List[Dict[str, Any]]] = None
+    ) -> List[str]:
+        """
+        Embed and store raw texts in the vector database.
+        
+        Args:
+            texts (List[str]): List of text strings
+            metadatas (List[Dict], optional): List of metadata dictionaries
+            
+        Returns:
+            List[str]: List of document IDs
+        """
+        if not texts:
+            logger.warning("No texts provided for embedding")
+            return []
+        
+        try:
+            # Filter metadata if provided
+            filtered_metadatas = None
+            if metadatas:
+                filtered_metadatas = []
+                for metadata in metadatas:
+                    filtered_metadata = self._filter_metadata(metadata)
+                    filtered_metadatas.append(filtered_metadata)
+                    
+                    # Log metadata transformation for debugging
+                    logger.debug(f"Original metadata keys: {list(metadata.keys())}")
+                    logger.debug(f"Filtered metadata keys: {list(filtered_metadata.keys())}")
+            
+            # Add texts to vector store
+            document_ids = self.vector_store.add_texts(
+                texts=texts,
+                metadatas=filtered_metadatas
+            )
+            
+            # ChromaDB auto-persists in newer versions
+            logger.info(f"Successfully embedded and stored {len(texts)} text documents")
+            return document_ids
+            
+        except Exception as e:
+            print(f"DEBUG: Error in embed_and_store_texts: {e}")
+            print(f"DEBUG: Error type: {type(e)}")
+            import traceback
+            traceback.print_exc()
+            logger.error(f"Failed to embed and store texts: {e}")
+            raise
+    
+    def similarity_search(
+        self, 
+        query: str, 
+        k: int = 4,
+        filter: Optional[Dict[str, Any]] = None
+    ) -> List[Document]:
+        """
+        Perform similarity search on stored documents.
+        
+        Args:
+            query (str): Search query
+            k (int): Number of results to return
+            filter (Dict, optional): Metadata filter
+            
+        Returns:
+            List[Document]: List of similar documents
+        """
+        try:
+            results = self.vector_store.similarity_search(
+                query=query,
+                k=k,
+                filter=filter
+            )
+            
+            logger.info(f"Found {len(results)} similar documents for query: '{query[:50]}...'")
+            return results
+            
+        except Exception as e:
+            logger.error(f"Failed to perform similarity search: {e}")
+            raise
+    
+    def similarity_search_with_score(
+        self, 
+        query: str, 
+        k: int = 4,
+        filter: Optional[Dict[str, Any]] = None
+    ) -> List[tuple]:
+        """
+        Perform similarity search with relevance scores.
+        
+        Args:
+            query (str): Search query
+            k (int): Number of results to return
+            filter (Dict, optional): Metadata filter
+            
+        Returns:
+            List[tuple]: List of (Document, score) tuples
+        """
+        try:
+            results = self.vector_store.similarity_search_with_score(
+                query=query,
+                k=k,
+                filter=filter
+            )
+            
+            logger.info(f"Found {len(results)} similar documents with scores for query: '{query[:50]}...'")
+            return results
+            
+        except Exception as e:
+            logger.error(f"Failed to perform similarity search with scores: {e}")
+            raise
+    
+    def get_collection_info(self) -> Dict[str, Any]:
+        """
+        Get information about the vector store collection.
+        
+        Returns:
+            Dict: Collection information
+        """
+        try:
+            collection = self.vector_store._collection
+            count = collection.count()
+            
+            return {
+                "collection_name": self.collection_name,
+                "document_count": count,
+                "persist_directory": self.persist_directory
+            }
+            
+        except Exception as e:
+            logger.error(f"Failed to get collection info: {e}")
+            return {}
+    
+    def delete_documents(self, ids: List[str]) -> bool:
+        """
+        Delete documents by their IDs.
+        
+        Args:
+            ids (List[str]): List of document IDs to delete
+            
+        Returns:
+            bool: Success status
+        """
+        try:
+            self.vector_store.delete(ids=ids)
+            # ChromaDB auto-persists in newer versions
+            
+            logger.info(f"Successfully deleted {len(ids)} documents")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to delete documents: {e}")
+            return False
+    
+    def delete_documents_by_metadata(self, filter_criteria: Dict[str, Any]) -> bool:
+        """
+        Delete documents based on metadata criteria.
+        
+        Args:
+            filter_criteria (Dict): Metadata criteria to filter documents for deletion
+            
+        Returns:
+            bool: Success status
+        """
+        try:
+            collection = self.vector_store._collection
+            
+            # Get all documents with their metadata
+            all_data = collection.get(include=['metadatas'])
+            ids_to_delete = []
+            
+            # Find documents that match the criteria
+            for i, metadata in enumerate(all_data['metadatas']):
+                match = True
+                for key, value in filter_criteria.items():
+                    if metadata.get(key) != value:
+                        match = False
+                        break
+                
+                if match:
+                    ids_to_delete.append(all_data['ids'][i])
+            
+            if ids_to_delete:
+                self.vector_store.delete(ids=ids_to_delete)
+                # ChromaDB auto-persists in newer versions
+                logger.info(f"Successfully deleted {len(ids_to_delete)} documents matching criteria: {filter_criteria}")
+                return True
+            else:
+                logger.info(f"No documents found matching criteria: {filter_criteria}")
+                return True
+                
+        except Exception as e:
+            logger.error(f"Failed to delete documents by metadata: {e}")
+            return False
+
+    def clear_collection(self) -> bool:
+        """
+        Clear all documents from the collection.
+        
+        Returns:
+            bool: Success status
+        """
+        try:
+            # Get all document IDs and delete them
+            collection = self.vector_store._collection
+            all_ids = collection.get()['ids']
+            
+            if all_ids:
+                self.vector_store.delete(ids=all_ids)
+                # ChromaDB auto-persists in newer versions
+                logger.info(f"Cleared {len(all_ids)} documents from collection")
+            else:
+                logger.info("Collection is already empty")
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to clear collection: {e}")
+            return False
+    
+    def batch_process_documents(
+        self, 
+        documents: List[Document], 
+        batch_size: int = 100
+    ) -> List[str]:
+        """
+        Process documents in batches for large datasets.
+        
+        Args:
+            documents (List[Document]): List of documents to process
+            batch_size (int): Size of each batch
+            
+        Returns:
+            List[str]: List of all document IDs
+        """
+        all_ids = []
+        
+        for i in range(0, len(documents), batch_size):
+            batch = documents[i:i + batch_size]
+            logger.info(f"Processing batch {i//batch_size + 1}/{(len(documents)-1)//batch_size + 1}")
+            
+            try:
+                batch_ids = self.embed_and_store_documents(batch)
+                all_ids.extend(batch_ids)
+            except Exception as e:
+                logger.error(f"Failed to process batch {i//batch_size + 1}: {e}")
+                continue
+        
+        logger.info(f"Completed batch processing. Total documents processed: {len(all_ids)}")
+        return all_ids
+
+class EmbeddingDemo:
+    def __init__(self, api_key=None):
+        """Initialize the embedding model with Google API key."""
+        if api_key:
+            os.environ["GOOGLE_API_KEY"] = api_key
+        
+        self.embeddings = GoogleGenerativeAIEmbeddings(
+            model="models/embedding-001"
+        )
+    
+    def embed_text(self, text):
+        """Generate embedding for a single text."""
+        return self.embeddings.embed_query(text)
+    
+    def embed_documents(self, documents):
+        """Generate embeddings for multiple documents."""
+        return self.embeddings.embed_documents(documents)
+    
+    def demonstrate(self):
+        """Show basic embedding functionality."""
+        sample_text = "This is a sample text for embedding."
+        sample_docs = ["First document", "Second document", "Third document"]
+        
+        # Single text embedding
+        text_embedding = self.embed_text(sample_text)
+        print(f"Text embedding dimension: {len(text_embedding)}")
+        
+        # Multiple documents embedding
+        doc_embeddings = self.embed_documents(sample_docs)
+        print(f"Number of document embeddings: {len(doc_embeddings)}")
+        print(f"Each embedding dimension: {len(doc_embeddings[0])}")
+
+# Usage example
+if __name__ == "__main__":
+    # Initialize the embedding and storage system
+    try:
+        # You need to set your Google API key
+        embedding_storage = DiaryEmbeddingAndStorage(
+            api_key="your_google_api_key_here",  # Replace with your actual API key
+            persist_directory="./diary_vector_db",
+            collection_name="diary_entries"
+        )
+        
+        # Example documents
+        sample_documents = [
+            Document(
+                page_content="Today was a wonderful day. I went to the park and enjoyed the sunshine.",
+                metadata={"date": "2024-01-15", "mood": "happy"}
+            ),
+            Document(
+                page_content="Had a challenging day at work but learned a lot of new things.",
+                metadata={"date": "2024-01-16", "mood": "productive"}
+            ),
+            Document(
+                page_content="Spent time with family and friends. Made some great memories.",
+                metadata={"date": "2024-01-17", "mood": "grateful"}
+            )
+        ]
+        
+        # Embed and store documents
+        doc_ids = embedding_storage.embed_and_store_documents(sample_documents)
+        print(f"Stored documents with IDs: {doc_ids}")
+        
+        # Get collection info
+        info = embedding_storage.get_collection_info()
+        print(f"Collection info: {info}")
+        
+        # Perform similarity search
+        query = "happy day at the park"
+        results = embedding_storage.similarity_search(query, k=2)
+        
+        print(f"\nSimilarity search results for '{query}':")
+        for i, doc in enumerate(results):
+            print(f"Result {i+1}: {doc.page_content[:100]}...")
+            print(f"Metadata: {doc.metadata}")
+        
+        # Search with scores
+        scored_results = embedding_storage.similarity_search_with_score(query, k=2)
+        
+        print(f"\nSimilarity search with scores:")
+        for doc, score in scored_results:
+            print(f"Score: {score:.4f} - {doc.page_content[:50]}...")
+        
+    except Exception as e:
+        print(f"Error in example: {e}")
+        
+    # Original demo
+    # demo = EmbeddingDemo(api_key="your_google_api_key_here")
+    # demo.demonstrate()

clean_repo/src/Indexingstep/indexing_pipeline.py

@@ -0,0 +1,110 @@
+import os
+import sys
+from typing import List, Dict, Any
+from datetime import datetime
+
+# Add parent directory to path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from langchain_chroma import Chroma
+from langchain.schema import Document
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+def create_user_vector_database(user_id: int, diary_entries: List[Dict[str, Any]]) -> bool:
+    """
+    Create vector database for a specific user from their diary entries.
+    
+    Args:
+        user_id: User ID
+        diary_entries: List of diary entries from database
+        
+    Returns:
+        True if successful, False otherwise
+    """
+    try:
+        # Setup paths
+        base_vector_path = os.path.dirname(os.path.abspath(__file__))
+        vector_db_path = os.path.join(base_vector_path, f"user_{user_id}_vector_db")
+        collection_name = f"user_{user_id}_diary_entries"
+        
+        # Create directory
+        os.makedirs(vector_db_path, exist_ok=True)
+        
+        # Initialize embeddings
+        google_api_key = os.getenv("GOOGLE_API_KEY")
+        if not google_api_key:
+            raise ValueError("Google API key not found")
+            
+        embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+        
+        # Process diary entries into documents
+        documents = []
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=1000,
+            chunk_overlap=200,
+            length_function=len,
+        )
+        
+        for entry in diary_entries:
+            # Extract content
+            content = entry.get('content', '')
+            if not content:
+                continue
+                
+            # Extract title and content
+            lines = content.split('\n')
+            title = "Untitled"
+            actual_content = content
+            
+            for line in lines:
+                if line.startswith('Title: '):
+                    title = line.replace('Title: ', '').strip()
+                elif line.startswith('Content: '):
+                    actual_content = line.replace('Content: ', '').strip()
+                    break
+            
+            # Create metadata
+            metadata = {
+                'user_id': user_id,
+                'entry_id': entry.get('id'),
+                'date': entry.get('date', ''),
+                'title': title,
+                'tags': entry.get('tags', ''),
+                'tags_list': [tag.strip() for tag in entry.get('tags', '').split(',') if tag.strip()],
+                'source': f"diary_entry_{entry.get('id')}"
+            }
+            
+            # Split content if too long
+            if len(actual_content) > 1000:
+                chunks = text_splitter.split_text(actual_content)
+                for i, chunk in enumerate(chunks):
+                    chunk_metadata = metadata.copy()
+                    chunk_metadata['chunk_id'] = i
+                    documents.append(Document(page_content=chunk, metadata=chunk_metadata))
+            else:
+                documents.append(Document(page_content=actual_content, metadata=metadata))
+        
+        if not documents:
+            print(f"No documents to index for user {user_id}")
+            return False
+        
+        # Create vector store
+        vector_store = Chroma(
+            persist_directory=vector_db_path,
+            embedding_function=embeddings,
+            collection_name=collection_name
+        )
+        
+        # Add documents to vector store
+        vector_store.add_documents(documents)
+        
+        # Persist the database
+        vector_store.persist()
+        
+        print(f"Successfully created vector database for user {user_id} with {len(documents)} documents")
+        return True
+        
+    except Exception as e:
+        print(f"Error creating vector database for user {user_id}: {e}")
+        return False

clean_repo/src/Indexingstep/pipeline.py

@@ -0,0 +1,459 @@
+import sys
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from dataloading import DiaryDataLoader, DiaryContentPreprocessor
+from diary_text_splitter import DiaryTextSplitter
+from embedding_and_storing import DiaryEmbeddingAndStorage
+from langchain.schema import Document
+from typing import List, Dict, Any, Optional
+import logging
+from pathlib import Path
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+class DiaryIndexingPipeline:
+    """
+    Enhanced pipeline for indexing diary entries with optimized chunking and metadata.
+    Integrates data loading, preprocessing, diary-specific splitting, embedding, and storage.
+    """
+    
+    def __init__(
+        self,
+        db_path: str = "./diary.db",
+        persist_directory: str = "./chroma_db",
+        collection_name: str = "diary_collection",
+        google_api_key: Optional[str] = None,
+        chunk_size: int = 300,  # Optimized for diary entries (200-300 tokens)
+        chunk_overlap: int = 50,  # 50-token sliding window
+        embedding_model: str = "models/embedding-001",
+        batch_size: int = 50,
+        user_id: int = 1
+    ):
+        """
+        Initialize the enhanced diary indexing pipeline.
+        
+        Args:
+            db_path (str): Path to SQLite database
+            persist_directory (str): Directory for vector database
+            collection_name (str): Name of the collection
+            google_api_key (str, optional): Google API key for embeddings
+            chunk_size (int): Size of text chunks (optimized for diary entries)
+            chunk_overlap (int): Overlap between chunks (sliding window)
+            embedding_model (str): Google embedding model name
+            batch_size (int): Batch size for processing
+            user_id (int): ID of the user for user-specific isolation
+        """
+        self.db_path = db_path
+        self.persist_directory = persist_directory
+        self.collection_name = collection_name
+        self.batch_size = batch_size
+        self.user_id = user_id
+        
+        # Validate database exists
+        if not os.path.exists(db_path):
+            raise FileNotFoundError(f"Database file not found: {db_path}")
+        
+        # Initialize components
+        self._initialize_components(
+            google_api_key, chunk_size, chunk_overlap, embedding_model
+        )
+        
+        logger.info("Diary Indexing Pipeline initialized successfully")
+    
+    def _initialize_components(
+        self, 
+        google_api_key: Optional[str], 
+        chunk_size: int, 
+        chunk_overlap: int,
+        embedding_model: str
+    ):
+        """Initialize all pipeline components."""
+        
+        # 1. Data Loader
+        self.data_loader = DiaryDataLoader(
+            db_path=self.db_path,
+            table_name="diary_entries",
+            content_column="content",
+            date_column="date",
+            user_id=self.user_id
+        )
+        
+        # 2. Content Preprocessor
+        self.preprocessor = DiaryContentPreprocessor(
+            remove_extra_whitespace=True,
+            normalize_line_breaks=True,
+            min_content_length=3,  # Keep short entries
+            max_content_length=10000
+        )
+        
+        # 3. Diary-optimized Text Splitter
+        self.text_splitter = DiaryTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap
+        )
+        
+        # 4. Embedding and Storage
+        self.embedding_storage = DiaryEmbeddingAndStorage(
+            user_id=self.user_id,
+            api_key=google_api_key,
+            base_persist_directory=self.persist_directory,
+            embedding_model=embedding_model,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap
+        )
+        
+        logger.info("All pipeline components initialized")
+    
+    def load_diary_data(self, start_date: Optional[str] = None, end_date: Optional[str] = None) -> List[Document]:
+        """
+        Load diary entries from database.
+        
+        Args:
+            start_date (str, optional): Start date filter (YYYY-MM-DD)
+            end_date (str, optional): End date filter (YYYY-MM-DD)
+            
+        Returns:
+            List[Document]: Loaded diary documents
+        """
+        try:
+            logger.info("Loading diary entries from database...")
+            
+            if start_date and end_date:
+                documents = self.data_loader.load_by_date_range(start_date, end_date)
+                logger.info(f"Loaded {len(documents)} entries from {start_date} to {end_date}")
+            else:
+                documents = self.data_loader.load()
+                logger.info(f"Loaded {len(documents)} total diary entries")
+            
+            if not documents:
+                logger.warning("No diary entries found in database")
+                return []
+            
+            return documents
+            
+        except Exception as e:
+            logger.error(f"Error loading diary data: {str(e)}")
+            raise
+    
+    def preprocess_documents(self, documents: List[Document]) -> List[Document]:
+        """
+        Preprocess diary documents.
+        
+        Args:
+            documents (List[Document]): Raw documents
+            
+        Returns:
+            List[Document]: Preprocessed documents
+        """
+        try:
+            logger.info(f"Preprocessing {len(documents)} documents...")
+            
+            preprocessed_docs = self.preprocessor.preprocess_documents(documents)
+            
+            logger.info(f"Preprocessing complete: {len(preprocessed_docs)} documents kept")
+            return preprocessed_docs
+            
+        except Exception as e:
+            logger.error(f"Error preprocessing documents: {str(e)}")
+            raise
+    
+    def split_documents(self, documents: List[Document]) -> List[Document]:
+        """
+        Split documents into optimized chunks using diary-specific splitter.
+        
+        Args:
+            documents (List[Document]): Documents to split
+            
+        Returns:
+            List[Document]: Split document chunks with enhanced metadata
+        """
+        try:
+            logger.info(f"Splitting {len(documents)} diary entries into optimized chunks...")
+            
+            split_docs = self.text_splitter.split_documents(documents)
+            
+            # Get and log chunking statistics
+            stats = self.text_splitter.get_chunk_stats(split_docs)
+            logger.info(f"Document splitting complete: {stats}")
+            
+            return split_docs
+            
+        except Exception as e:
+            logger.error(f"Error splitting documents: {str(e)}")
+            raise
+    
+    def embed_and_store(self, documents: List[Document]) -> List[str]:
+        """
+        Generate embeddings and store documents.
+        
+        Args:
+            documents (List[Document]): Documents to embed and store
+            
+        Returns:
+            List[str]: Document IDs
+        """
+        try:
+            logger.info(f"Generating embeddings and storing {len(documents)} document chunks...")
+            
+            # Process in batches for large datasets
+            if len(documents) > self.batch_size:
+                document_ids = self.embedding_storage.batch_process_documents(
+                    documents, self.batch_size
+                )
+            else:
+                document_ids = self.embedding_storage.embed_and_store_documents(documents)
+            
+            logger.info(f"Successfully embedded and stored {len(document_ids)} documents")
+            return document_ids
+            
+        except Exception as e:
+            logger.error(f"Error embedding and storing documents: {str(e)}")
+            raise
+    
+    def run_full_pipeline(
+        self, 
+        start_date: Optional[str] = None, 
+        end_date: Optional[str] = None,
+        clear_existing: bool = False
+    ) -> Dict[str, Any]:
+        """
+        Run the complete indexing pipeline.
+        
+        Args:
+            start_date (str, optional): Start date filter
+            end_date (str, optional): End date filter
+            clear_existing (bool): Whether to clear existing data
+            
+        Returns:
+            Dict: Pipeline execution results
+        """
+        try:
+            logger.info("="*60)
+            logger.info("STARTING DIARY INDEXING PIPELINE")
+            logger.info("="*60)
+            
+            pipeline_stats = {
+                "status": "running",
+                "steps_completed": 0,
+                "total_steps": 5,
+                "documents_loaded": 0,
+                "documents_preprocessed": 0,
+                "chunks_created": 0,
+                "documents_stored": 0,
+                "errors": []
+            }
+            
+            # Step 1: Clear existing data if requested
+            if clear_existing:
+                logger.info("Step 1: Clearing existing vector store...")
+                self.embedding_storage.clear_collection()
+                pipeline_stats["steps_completed"] += 1
+            
+            # Step 2: Load diary data
+            logger.info("Step 2: Loading diary entries...")
+            documents = self.load_diary_data(start_date, end_date)
+            pipeline_stats["documents_loaded"] = len(documents)
+            pipeline_stats["steps_completed"] += 1
+            
+            if not documents:
+                pipeline_stats["status"] = "completed_with_warnings"
+                pipeline_stats["errors"].append("No documents found to process")
+                return pipeline_stats
+            
+            # Step 3: Preprocess documents
+            logger.info("Step 3: Preprocessing documents...")
+            preprocessed_docs = self.preprocess_documents(documents)
+            pipeline_stats["documents_preprocessed"] = len(preprocessed_docs)
+            pipeline_stats["steps_completed"] += 1
+            
+            if not preprocessed_docs:
+                pipeline_stats["status"] = "failed"
+                pipeline_stats["errors"].append("No documents survived preprocessing")
+                return pipeline_stats
+            
+            # Step 4: Split documents into chunks
+            logger.info("Step 4: Splitting documents into chunks...")
+            split_docs = self.split_documents(preprocessed_docs)
+            pipeline_stats["chunks_created"] = len(split_docs)
+            pipeline_stats["steps_completed"] += 1
+            
+            # Step 5: Generate embeddings and store
+            logger.info("Step 5: Generating embeddings and storing...")
+            document_ids = self.embed_and_store(split_docs)
+            pipeline_stats["documents_stored"] = len(document_ids)
+            pipeline_stats["steps_completed"] += 1
+            
+            # Update final status
+            pipeline_stats["status"] = "completed_successfully"
+            
+            logger.info("="*60)
+            logger.info("PIPELINE COMPLETED SUCCESSFULLY!")
+            logger.info("="*60)
+            logger.info(f"Documents loaded: {pipeline_stats['documents_loaded']}")
+            logger.info(f"Documents preprocessed: {pipeline_stats['documents_preprocessed']}")
+            logger.info(f"Chunks created: {pipeline_stats['chunks_created']}")
+            logger.info(f"Documents stored: {pipeline_stats['documents_stored']}")
+            logger.info("="*60)
+            
+            return pipeline_stats
+            
+        except Exception as e:
+            logger.error(f"Pipeline failed with error: {str(e)}")
+            pipeline_stats["status"] = "failed"
+            pipeline_stats["errors"].append(str(e))
+            return pipeline_stats
+    
+    def incremental_update(self, start_date: str, end_date: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Perform incremental update for new diary entries.
+        
+        Args:
+            start_date (str): Start date for incremental update
+            end_date (str, optional): End date for incremental update
+            
+        Returns:
+            Dict: Update results
+        """
+        try:
+            logger.info(f"Starting incremental update from {start_date}")
+            
+            # Load only new entries
+            new_documents = self.load_diary_data(start_date, end_date)
+            
+            if not new_documents:
+                logger.info("No new documents found for incremental update")
+                return {"status": "no_updates", "documents_added": 0}
+            
+            # Process new documents
+            preprocessed_docs = self.preprocess_documents(new_documents)
+            split_docs = self.split_documents(preprocessed_docs)
+            document_ids = self.embed_and_store(split_docs)
+            
+            logger.info(f"Incremental update completed: {len(document_ids)} new documents added")
+            
+            return {
+                "status": "success",
+                "documents_loaded": len(new_documents),
+                "documents_added": len(document_ids)
+            }
+            
+        except Exception as e:
+            logger.error(f"Incremental update failed: {str(e)}")
+            return {"status": "failed", "error": str(e)}
+    
+    def search_similar_entries(
+        self, 
+        query: str, 
+        k: int = 5,
+        filter_metadata: Optional[Dict[str, Any]] = None
+    ) -> List[Document]:
+        """
+        Search for similar diary entries.
+        
+        Args:
+            query (str): Search query
+            k (int): Number of results to return
+            filter_metadata (Dict, optional): Metadata filter
+            
+        Returns:
+            List[Document]: Similar documents
+        """
+        try:
+            return self.embedding_storage.similarity_search(
+                query=query,
+                k=k,
+                filter=filter_metadata
+            )
+        except Exception as e:
+            logger.error(f"Error searching similar entries: {str(e)}")
+            return []
+    
+    def get_pipeline_stats(self) -> Dict[str, Any]:
+        """
+        Get comprehensive pipeline statistics.
+        
+        Returns:
+            Dict: Pipeline and database statistics
+        """
+        try:
+            # Database stats
+            db_info = self.data_loader.get_table_info()
+            
+            # Vector store stats
+            vector_info = self.embedding_storage.get_collection_info()
+            
+            return {
+                "database": db_info,
+                "vector_store": vector_info,
+                "pipeline_config": {
+                    "chunk_size": self.text_splitter.chunk_size,
+                    "chunk_overlap": self.text_splitter.chunk_overlap,
+                    "batch_size": self.batch_size,
+                    "collection_name": self.collection_name
+                }
+            }
+            
+        except Exception as e:
+            logger.error(f"Error getting pipeline stats: {str(e)}")
+            return {}
+
+def main():
+    """Main function to demonstrate pipeline usage."""
+    
+    # Configuration
+    config = {
+        "db_path": "../streamlit_app/backend/diary.db",  # Adjust path as needed
+        "persist_directory": "./diary_vector_db",
+        "collection_name": "diary_entries",
+        "google_api_key": None,  # Set your API key or use environment variable
+        "chunk_size": 800,
+        "chunk_overlap": 100,
+        "batch_size": 50
+    }
+    
+    try:
+        # Initialize pipeline
+        logger.info("Initializing Diary Indexing Pipeline...")
+        pipeline = DiaryIndexingPipeline(**config)
+        
+        # Run full pipeline
+        results = pipeline.run_full_pipeline(clear_existing=True)
+        
+        # Print results
+        print("\n" + "="*60)
+        print("PIPELINE EXECUTION RESULTS")
+        print("="*60)
+        print(f"Status: {results['status']}")
+        print(f"Steps completed: {results['steps_completed']}/{results['total_steps']}")
+        print(f"Documents loaded: {results['documents_loaded']}")
+        print(f"Documents preprocessed: {results['documents_preprocessed']}")
+        print(f"Chunks created: {results['chunks_created']}")
+        print(f"Documents stored: {results['documents_stored']}")
+        
+        if results['errors']:
+            print(f"Errors: {results['errors']}")
+        
+        # Get and display stats
+        stats = pipeline.get_pipeline_stats()
+        print("\nPIPELINE STATISTICS:")
+        print(f"Database entries: {stats.get('database', {}).get('row_count', 'N/A')}")
+        print(f"Vector store documents: {stats.get('vector_store', {}).get('document_count', 'N/A')}")
+        print("="*60)
+        
+        # Example search
+        if results['status'] == 'completed_successfully':
+            print("\nTesting similarity search...")
+            search_results = pipeline.search_similar_entries("happy day", k=3)
+            print(f"Found {len(search_results)} similar entries")
+            for i, doc in enumerate(search_results[:2]):
+                print(f"Result {i+1}: {doc.page_content[:100]}...")
+        
+    except Exception as e:
+        logger.error(f"Main execution failed: {str(e)}")
+        print(f"Error: {str(e)}")
+
+if __name__ == "__main__":
+    main()

clean_repo/src/Retrivel_And_Generation/Retrieval_And_Generator.py

@@ -0,0 +1,739 @@
+#!/usr/bin/env python3
+"""
+Retrieval and Generation System for Personal Diary Chatbot
+
+This module implements the RAG (Retrieval-Augmented Generation) pipeline for the diary chatbot.
+It handles document retrieval from the vector database and generates contextual responses
+using Google's Generative AI.
+
+Components:
+- Document Retrieval: Query vector database for relevant diary entries
+- Context Processing: Format retrieved documents for LLM consumption
+- Response Generation: Generate contextual responses using retrieved diary content
+- Conversation Management: Handle chat history and context preservation
+"""
+
+import os
+import sys
+import logging
+from typing import List, Dict, Any, Optional, Tuple
+from datetime import datetime
+from functools import lru_cache
+import hashlib
+
+# Add parent directory to path for imports
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+# LangChain imports
+from langchain_google_genai import GoogleGenerativeAIEmbeddings, ChatGoogleGenerativeAI
+from langchain_chroma import Chroma
+from langchain.schema import Document
+from langchain.schema.runnable import RunnablePassthrough
+from langchain.schema.output_parser import StrOutputParser
+from langchain.prompts import ChatPromptTemplate, PromptTemplate
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class DiaryRAGSystem:
+    """
+    Retrieval-Augmented Generation system for personal diary chatbot.
+    
+    This class handles the complete RAG pipeline:
+    1. Retrieve relevant diary entries from vector database
+    2. Format context for LLM consumption
+    3. Generate contextual responses using Google's Generative AI
+    """
+    
+    def __init__(
+        self,
+        user_id: int = 1,
+        base_vector_path: str = "./src/VectorDB",
+        google_api_key: Optional[str] = None,
+        embedding_model: str = "models/embedding-001",
+        chat_model: str = "gemini-2.5-flash-lite",
+        max_retrieval_docs: int = 5
+    ):
+        """
+        Initialize the RAG system with user-specific vector database.
+        
+        Args:
+            user_id: User ID for user-specific vector database
+            base_vector_path: Base path for vector databases
+            google_api_key: Google API key for embeddings and chat
+            embedding_model: Model for text embeddings
+            chat_model: Model for chat completion
+            max_retrieval_docs: Maximum number of documents to retrieve
+        """
+        self.user_id = user_id
+        self.base_vector_path = base_vector_path
+        
+        # Create user-specific paths
+        self.vector_db_path = os.path.join(base_vector_path, f"user_{user_id}_vector_db")
+        self.collection_name = f"user_{user_id}_diary_entries"
+        self.max_retrieval_docs = max_retrieval_docs
+        
+        # Ensure user vector database directory exists
+        os.makedirs(self.vector_db_path, exist_ok=True)
+        
+        # Set up Google API key
+        if google_api_key:
+            os.environ["GOOGLE_API_KEY"] = google_api_key
+        elif not os.getenv("GOOGLE_API_KEY"):
+            raise ValueError("Google API key must be provided either as parameter or environment variable")
+        
+        # Initialize embedding and chat models
+        try:
+            # Fix for Streamlit event loop issue
+            import asyncio
+            import nest_asyncio
+            
+            # Allow nested event loops for Streamlit compatibility
+            try:
+                nest_asyncio.apply()
+            except:
+                pass
+                
+            # Set event loop for thread if not exists
+            try:
+                loop = asyncio.get_event_loop()
+                if loop.is_closed():
+                    raise RuntimeError("Event loop is closed")
+            except RuntimeError:
+                # Create new event loop for this thread
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+            
+            self.embeddings = GoogleGenerativeAIEmbeddings(model=embedding_model)
+            self.chat_model = ChatGoogleGenerativeAI(
+                model=chat_model,
+                temperature=0.3,  # Lower temperature for faster, more focused responses
+                max_tokens=800,   # Shorter responses for speed
+                top_k=20,        # Limit token choices for speed
+                top_p=0.8        # Nucleus sampling for faster generation
+            )
+            logger.info(f"Initialized embeddings with model: {embedding_model}")
+            logger.info(f"Initialized chat model: {chat_model}")
+        except Exception as e:
+            logger.error(f"Failed to initialize models: {str(e)}")
+            raise
+        
+        # Initialize vector store
+        self.vector_store = None
+        self._setup_vector_store()
+        
+        # Set up prompt templates
+        self._setup_prompts()
+        
+        # Initialize conversation chain
+        self._setup_conversation_chain()
+    
+    def _setup_vector_store(self):
+        """Set up connection to the vector database."""
+        try:
+            if os.path.exists(self.vector_db_path):
+                self.vector_store = Chroma(
+                    persist_directory=self.vector_db_path,
+                    embedding_function=self.embeddings,
+                    collection_name=self.collection_name
+                )
+                collection_info = self.vector_store._collection.count()
+                logger.info(f"Connected to vector database (primary) with {collection_info} documents")
+                # Fallback: legacy nested path if empty
+                if collection_info == 0:
+                    nested_path = os.path.join(self.vector_db_path, os.path.basename(self.vector_db_path))
+                    if os.path.isdir(nested_path):
+                        try:
+                            nested_vs = Chroma(
+                                persist_directory=nested_path,
+                                embedding_function=self.embeddings,
+                                collection_name=self.collection_name
+                            )
+                            nested_count = nested_vs._collection.count()
+                            if nested_count > 0:
+                                logger.warning(
+                                    f"Primary path empty. Switching to legacy nested path {nested_path} with {nested_count} docs"
+                                )
+                                self.vector_store = nested_vs
+                                self.vector_db_path = nested_path
+                        except Exception as ne:
+                            logger.debug(f"Failed to read nested path: {ne}")
+            else:
+                logger.warning(f"Vector database not found at {self.vector_db_path}")
+                logger.info("Run indexing pipeline first.")
+        except Exception as e:
+            logger.error(f"Failed to setup vector store: {str(e)}")
+            self.vector_store = None
+
+    def reload_vector_store(self) -> int:
+        """Reload vector store from disk. Returns new document count or 0."""
+        try:
+            self._setup_vector_store()
+            if self.vector_store:
+                return self.vector_store._collection.count()
+        except Exception as e:
+            logger.warning(f"reload_vector_store failed: {e}")
+        return 0
+
+    def get_document_count(self) -> int:
+        try:
+            if self.vector_store:
+                return self.vector_store._collection.count()
+        except Exception:
+            pass
+        return 0
+    
+    def _setup_prompts(self):
+        """Set up prompt templates for different scenarios."""
+        
+        # Main RAG prompt template
+        self.rag_prompt = ChatPromptTemplate.from_template("""
+Bạn là một trợ lý AI thông minh và thấu hiểu, chuyên về việc phân tích và thảo luận nội dung về nhật ký cá nhân.
+
+Dựa trên các mục nhật ký sau đây được tìm kiếm từ cơ sở dữ liệu:
+
+{context}
+
+Người dùng hỏi: {question}
+
+Hãy trả lời một cách:
+- Thấu hiểu và empathetic (đồng cảm)
+- Dựa trên nội dung nhật ký được cung cấp
+- Cung cấp insights và connections giữa các entries
+- Đưa ra suggestions hoặc reflections nếu phù hợp
+- Sử dụng tiếng Việt tự nhiên và ấm áp
+
+Nếu không tìm thấy thông tin liên quan trong nhật ký, hãy thành thật nói và đề xuất các cách khác để giúp đỡ.
+
+Trả lời:
+""")
+        
+        # Fallback prompt when no relevant documents found
+        self.fallback_prompt = ChatPromptTemplate.from_template("""
+Bạn là một trợ lý AI thân thiện và hữu ích cho việc quản lý nhật ký cá nhân.
+
+Người dùng hỏi: {question}
+
+Vì không tìm thấy thông tin liên quan trong nhật ký hiện tại, hãy:
+- Trả lời một cách thân thiện ngắn gọn và hữu ích
+- Đề xuất cách người dùng có thể ghi nhật ký về chủ đề này
+- Khuyến khích reflection và self-discovery
+- Cung cấp general guidance nếu phù hợp
+
+Sử dụng tiếng Việt tự nhiên và ấm áp.
+
+Trả lời:
+""")
+        
+        # Summary prompt for multiple diary entries
+        self.summary_prompt = ChatPromptTemplate.from_template("""
+Dựa trên các mục nhật ký sau đây:
+
+{context}
+
+Hãy tạo một summary ngắn gọn về:
+- Chủ đề chính được đề cập
+- Cảm xúc và mood tổng thể
+- Patterns hoặc themes đáng chú ý
+- Insights về personal growth
+
+Sử dụng tiếng Việt và giữ tính cách empathetic.
+
+Summary:
+""")
+    
+    def _setup_conversation_chain(self):
+        """Set up the conversation chain for RAG processing."""
+        try:
+            # Create retriever from vector store
+            if self.vector_store:
+                self.retriever = self.vector_store.as_retriever(
+                    search_kwargs={"k": self.max_retrieval_docs}
+                )
+                
+                # Set up the main RAG chain
+                self.rag_chain = (
+                    {
+                        "context": self.retriever | self._format_docs,
+                        "question": RunnablePassthrough()
+                    }
+                    | self.rag_prompt
+                    | self.chat_model
+                    | StrOutputParser()
+                )
+                
+                # Set up fallback chain
+                self.fallback_chain = (
+                    {"question": RunnablePassthrough()}
+                    | self.fallback_prompt
+                    | self.chat_model
+                    | StrOutputParser()
+                )
+                
+                logger.info("Conversation chain setup complete")
+            else:
+                logger.warning("Cannot setup conversation chain without vector store")
+                
+        except Exception as e:
+            logger.error(f"Failed to setup conversation chain: {str(e)}")
+            raise
+    
+    def _format_docs(self, docs: List[Document]) -> str:
+        """
+        Format retrieved documents for LLM consumption.
+        
+        Args:
+            docs: List of retrieved documents
+            
+        Returns:
+            Formatted string with document content and metadata
+        """
+        if not docs:
+            return "Không tìm thấy mục nhật ký liên quan."
+        
+        formatted_docs = []
+        for i, doc in enumerate(docs, 1):
+            # Extract metadata
+            metadata = doc.metadata
+            date = metadata.get('date', 'Unknown date')
+            title = metadata.get('title', 'Untitled')
+            tags = metadata.get('tags_list', metadata.get('tags', ''))
+            
+            # Format document
+            doc_text = f"""
+Mục {i}:
+Ngày: {date}
+Tiêu đề: {title}
+Tags: {tags if tags else 'Không có tags'}
+Nội dung: {doc.page_content.strip()}
+---
+"""
+            formatted_docs.append(doc_text)
+        
+        return "\n".join(formatted_docs)
+    
+    def retrieve_relevant_entries(
+        self, 
+        query: str, 
+        filters: Optional[Dict[str, Any]] = None,
+        k: Optional[int] = None
+    ) -> List[Document]:
+        """
+        Retrieve relevant diary entries based on query with optimized performance.
+        
+        Args:
+            query: Search query
+            filters: Optional metadata filters
+            k: Number of documents to retrieve (overrides default)
+            
+        Returns:
+            List of relevant documents
+        """
+        if not self.vector_store:
+            logger.warning("Vector store not available for retrieval")
+            return []
+        
+        try:
+            # Use smaller k for faster response
+            k = k or min(self.max_retrieval_docs, 3)  # Limit to 3 docs for speed
+            
+            if filters:
+                docs = self.vector_store.similarity_search(
+                    query=query,
+                    k=k,
+                    filter=filters
+                )
+            else:
+                docs = self.vector_store.similarity_search(
+                    query=query,
+                    k=k
+                )
+            
+            logger.info(f"Retrieved {len(docs)} documents for query: '{query[:50]}...'")
+            return docs
+            
+        except Exception as e:
+            logger.error(f"Error during retrieval: {str(e)}")
+            return []
+    
+    def format_documents_for_context(self, docs: List[Document]) -> str:
+        """
+        Format retrieved documents into context string for the prompt.
+        
+        Args:
+            docs: List of retrieved documents
+            
+        Returns:
+            Formatted context string
+        """
+        if not docs:
+            return "Không có thông tin nhật ký liên quan."
+        
+        formatted_docs = []
+        for i, doc in enumerate(docs, 1):
+            # Extract metadata
+            metadata = doc.metadata
+            date = metadata.get('date', 'Không có ngày')
+            source = metadata.get('source', 'Không rõ nguồn')
+            
+            # Format document
+            doc_text = f"Nhật ký {i} (Ngày: {date}):\n{doc.page_content}"
+            formatted_docs.append(doc_text)
+        
+        return "\n\n".join(formatted_docs)
+    
+    def generate_fast_response(
+        self, 
+        query: str, 
+        filters: Optional[Dict[str, Any]] = None
+    ) -> str:
+        """
+        Generate fast response with optimized settings for speed.
+        
+        Args:
+            query: User question
+            filters: Optional metadata filters
+            
+        Returns:
+            AI response string (optimized for speed)
+        """
+        try:
+            # Fast retrieval with only 1 most relevant doc for maximum speed
+            relevant_docs = self.retrieve_relevant_entries(
+                query=query, 
+                filters=filters,
+                k=1  # Only 1 doc for maximum speed
+            )
+            
+            if not relevant_docs:
+                # Use simple fallback without chain to avoid timeout
+                return "Xin lỗi, tôi không tìm thấy thông tin liên quan trong nhật ký của bạn."
+            
+            # Create very concise context (limit content length)
+            context = self._format_docs(relevant_docs[:1])
+            if len(context) > 500:  # Limit context length
+                context = context[:500] + "..."
+            
+            # Fast prompt template with timeout optimization
+            fast_prompt = ChatPromptTemplate.from_template(
+                """Dựa vào nhật ký: {context}
+
+Câu hỏi: {question}
+
+Trả lời ngắn (1 câu):"""
+            )
+            
+            # Create optimized chain with pre-computed context
+            chain = (
+                {"context": lambda x: context, "question": RunnablePassthrough()}
+                | fast_prompt
+                | self.chat_model
+                | StrOutputParser()
+            )
+            
+            # Generate response with timeout handling
+            response = chain.invoke(query)
+            logger.info("Generated fast response successfully")
+            return response.strip()
+            
+        except Exception as e:
+            logger.error(f"Error in fast response generation: {str(e)}")
+            # Direct fallback without chain to avoid timeout
+            return "Xin lỗi, tôi gặp lỗi khi xử lý câu hỏi của bạn."
+
+    def generate_response(
+        self, 
+        query: str, 
+        filters: Optional[Dict[str, Any]] = None,
+        use_fallback: bool = False
+    ) -> str:
+        """
+        Generate a response to user query using RAG.
+        
+        Args:
+            query: User's question or message
+            filters: Optional metadata filters for retrieval
+            use_fallback: Whether to use fallback response (no retrieval)
+            
+        Returns:
+            Generated response
+        """
+        try:
+            if use_fallback or not self.vector_store:
+                # Use fallback chain without retrieval
+                response = self.fallback_chain.invoke(query)
+                logger.info("Generated fallback response")
+                return response
+            
+            # Retrieve relevant documents first
+            relevant_docs = self.retrieve_relevant_entries(query, filters)
+            
+            if not relevant_docs:
+                # No relevant documents found, use fallback
+                response = self.fallback_chain.invoke(query)
+                logger.info("No relevant docs found, used fallback response")
+                return response
+            
+            # Use RAG chain with retrieved context
+            response = self.rag_chain.invoke(query)
+            logger.info("Generated RAG response with context")
+            return response
+            
+        except Exception as e:
+            logger.error(f"Error generating response: {str(e)}")
+            return f"Xin lỗi, tôi gặp lỗi khi xử lý câu hỏi của bạn: {str(e)}"
+    
+    def generate_summary(self, date_range: Optional[Tuple[str, str]] = None) -> str:
+        """
+        Generate a summary of diary entries.
+        
+        Args:
+            date_range: Optional tuple of (start_date, end_date) in YYYY-MM-DD format
+            
+        Returns:
+            Generated summary
+        """
+        try:
+            if not self.vector_store:
+                return "Không thể tạo summary: vector database không khả dụng."
+            
+            # Build filter for date range if provided
+            filters = {}
+            if date_range:
+                start_date, end_date = date_range
+                # Note: This depends on how dates are stored in metadata
+                # May need adjustment based on actual metadata structure
+                pass
+            
+            # Retrieve documents for summary (more documents for better overview)
+            docs = self.vector_store.similarity_search(
+                query="nhật ký cảm xúc thoughts feelings",  # General query
+                k=min(10, self.max_retrieval_docs * 2)  # More docs for summary
+            )
+            
+            if not docs:
+                return "Không tìm thấy nh��t ký để tạo summary."
+            
+            # Format context for summary
+            context = self._format_docs(docs)
+            
+            # Generate summary
+            summary_chain = (
+                {"context": lambda x: context}
+                | self.summary_prompt
+                | self.chat_model
+                | StrOutputParser()
+            )
+            
+            summary = summary_chain.invoke({})
+            logger.info("Generated diary summary")
+            return summary
+            
+        except Exception as e:
+            logger.error(f"Error generating summary: {str(e)}")
+            return f"Lỗi khi tạo summary: {str(e)}"
+    
+    def search_by_tags(self, tags: List[str], k: int = 5) -> List[Document]:
+        """
+        Search diary entries by specific tags.
+        
+        Args:
+            tags: List of tags to search for
+            k: Number of documents to return
+            
+        Returns:
+            List of documents matching the tags
+        """
+        if not self.vector_store or not tags:
+            return []
+        
+        try:
+            # Build tag query
+            tag_query = " ".join([f"#{tag}" for tag in tags])
+            
+            # Search with tag-based query
+            docs = self.vector_store.similarity_search(
+                query=tag_query,
+                k=k
+            )
+            
+            # Filter by tags in metadata if available
+            filtered_docs = []
+            for doc in docs:
+                doc_tags = doc.metadata.get('tags_list', '')
+                if any(tag.lower() in doc_tags.lower() for tag in tags):
+                    filtered_docs.append(doc)
+            
+            logger.info(f"Found {len(filtered_docs)} documents with tags: {tags}")
+            return filtered_docs
+            
+        except Exception as e:
+            logger.error(f"Error searching by tags: {str(e)}")
+            return []
+    
+    def get_conversation_context(self, chat_history: List[Dict[str, str]]) -> str:
+        """
+        Process chat history to maintain conversation context.
+        
+        Args:
+            chat_history: List of chat messages with 'role' and 'content'
+            
+        Returns:
+            Formatted conversation context
+        """
+        if not chat_history:
+            return ""
+        
+        # Take last few messages for context
+        recent_messages = chat_history[-5:]  # Last 5 messages
+        
+        context_parts = []
+        for msg in recent_messages:
+            role = "Người dùng" if msg['role'] == 'user' else "Trợ lý"
+            context_parts.append(f"{role}: {msg['content']}")
+        
+        return "\n".join(context_parts)
+    
+    def generate_contextual_response(
+        self, 
+        query: str, 
+        chat_history: List[Dict[str, str]] = None,
+        filters: Optional[Dict[str, Any]] = None
+    ) -> str:
+        """
+        Generate response with conversation context.
+        
+        Args:
+            query: Current user query
+            chat_history: Previous conversation messages
+            filters: Optional metadata filters
+            
+        Returns:
+            Contextual response
+        """
+        # Get conversation context
+        conv_context = self.get_conversation_context(chat_history or [])
+        
+        # Enhance query with conversation context
+        if conv_context:
+            enhanced_query = f"Bối cảnh cuộc trò chuyện:\n{conv_context}\n\nCâu hỏi hiện tại: {query}"
+        else:
+            enhanced_query = query
+        
+        # Generate response
+        return self.generate_response(enhanced_query, filters)
+    
+    def health_check(self) -> Dict[str, Any]:
+        """
+        Check the health status of the RAG system.
+        
+        Returns:
+            Dictionary with system status information
+        """
+        status = {
+            "vector_store_available": self.vector_store is not None,
+            "vector_db_path": self.vector_db_path,
+            "models_initialized": True,
+            "embedding_model": "models/embedding-001",
+            "chat_model": "gemini-1.5-flash"
+        }
+        
+        if self.vector_store:
+            try:
+                doc_count = self.vector_store._collection.count()
+                status["document_count"] = doc_count
+                status["vector_store_healthy"] = True
+            except Exception as e:
+                status["vector_store_healthy"] = False
+                status["vector_store_error"] = str(e)
+        else:
+            status["document_count"] = 0
+            status["vector_store_healthy"] = False
+        
+        return status
+
+# ========================================
+# CONVENIENCE FUNCTIONS
+# ========================================
+
+def create_rag_system(
+    user_id: int = 1,
+    base_vector_path: str = "./src/Indexingstep",
+    google_api_key: Optional[str] = None
+) -> DiaryRAGSystem:
+    """
+    Create and initialize a user-specific DiaryRAGSystem instance.
+    
+    Args:
+        user_id: User ID for user-specific vector database
+        base_vector_path: Base path for vector databases
+        google_api_key: Google API key
+        
+    Returns:
+        Initialized DiaryRAGSystem for the specific user
+    """
+    return DiaryRAGSystem(
+        user_id=user_id,
+        base_vector_path=base_vector_path,
+        google_api_key=google_api_key
+    )
+
+def quick_query(
+    query: str, 
+    user_id: int = 1,
+    base_vector_path: str = "./src/VectorDB"
+) -> str:
+    """
+    Quick query function for testing with user-specific database.
+    
+    Args:
+        query: Question to ask
+        user_id: User ID for user-specific vector database
+        base_vector_path: Base path for vector databases
+        
+    Returns:
+        Response string
+    """
+    try:
+        rag = create_rag_system(user_id, base_vector_path)
+        return rag.generate_response(query)
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+if __name__ == "__main__":
+    # Example usage
+    print("🤖 Diary RAG System - Example Usage")
+    print("=" * 50)
+    
+    try:
+        # Initialize system
+        rag = create_rag_system()
+        
+        # Health check
+        status = rag.health_check()
+        print("System Status:")
+        for key, value in status.items():
+            print(f"  {key}: {value}")
+        
+        # Example queries
+        if status.get("vector_store_healthy"):
+            print("\n📝 Example Queries:")
+            
+            queries = [
+                "Tôi cảm thấy như thế nào trong tuần này?",
+                "Có những hoạt động nào tôi đã làm gần đây?",
+                "Tâm trạng của tôi đã thay đổi như thế nào?"
+            ]
+            
+            for query in queries:
+                print(f"\n❓ Query: {query}")
+                response = rag.generate_response(query)
+                print(f"🤖 Response: {response[:200]}...")
+        
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+        print("Make sure to:")
+        print("1. Set GOOGLE_API_KEY environment variable")
+        print("2. Run the indexing pipeline first")
+        print("3. Check vector database path")

clean_repo/src/rag_service/main.py

@@ -0,0 +1,721 @@
+from fastapi import FastAPI, HTTPException, BackgroundTasks
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import List, Dict, Any, Optional
+import os
+import sys
+import uvicorn
+from datetime import datetime
+import json
+import logging
+from fastapi import Query
+
+# Load environment variables
+from dotenv import load_dotenv
+load_dotenv()
+
+# Add paths for imports
+current_dir = os.path.dirname(os.path.abspath(__file__))
+src_dir = os.path.dirname(current_dir)
+sys.path.append(src_dir)
+sys.path.append(os.path.join(src_dir, "Indexingstep"))
+sys.path.append(os.path.join(src_dir, "Retrivel_And_Generation"))
+
+# Import your modules
+try:
+    from Indexingstep.pipeline import DiaryIndexingPipeline
+    from Retrivel_And_Generation.Retrieval_And_Generator import create_rag_system, DiaryRAGSystem
+    RAG_MODULES_AVAILABLE = True
+except ImportError as e:
+    print(f"Warning: RAG modules not available: {e}")
+    RAG_MODULES_AVAILABLE = False
+
+# Configure logging
+logging.basicConfig(filename="logs/service.log",
+                    level=logging.INFO )
+logger = logging.getLogger(__name__)
+
+app = FastAPI(
+    title="Personal Diary RAG Service",
+    description="RAG service for personal diary chatbot with user isolation",
+    version="1.0.0"
+)
+
+# CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# In-memory cache for RAG systems
+rag_systems_cache: Dict[int, DiaryRAGSystem] = {}
+
+# ========================================
+# PYDANTIC MODELS
+# ========================================
+
+class DiaryEntry(BaseModel):
+    date: str
+    content: str
+    tags: str = ""
+
+class IndexRequest(BaseModel):
+    user_id: int
+    clear_existing: bool = False
+    start_date: Optional[str] = None
+    end_date: Optional[str] = None
+
+class QueryRequest(BaseModel):
+    user_id: int
+    query: str
+    fast_mode: bool = False
+    chat_history: List[Dict[str, str]] = []
+
+class UserStatusResponse(BaseModel):
+    user_id: int
+    status: str
+    document_count: int
+    vector_db_path: str
+    last_updated: Optional[str] = None
+    error: Optional[str] = None
+
+class QueryResponse(BaseModel):
+    user_id: int
+    response: str
+    processing_time: float
+    documents_used: int
+    fast_mode: bool
+
+class IndexResponse(BaseModel):
+    user_id: int
+    status: str
+    documents_processed: int
+    chunks_created: int
+    vector_db_path: str
+    processing_time: float
+    error: Optional[str] = None
+
+# ========================================
+# HELPER FUNCTIONS
+# ========================================
+
+def format_error_message(errors) -> str:
+    """Convert error list to string for API response."""
+    if isinstance(errors, list):
+        return '; '.join(str(e) for e in errors)
+    return str(errors) if errors else 'Unknown error'
+
+def get_user_paths(user_id: int) -> Dict[str, str]:
+    """Get all paths for a user."""
+    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    
+    return {
+        "vector_db_path": os.path.join(base_dir, "VectorDB", f"user_{user_id}_vector_db"),
+        "diary_db_path": os.path.join(base_dir, "streamlit_app", "backend", f"user_{user_id}_diary.db"),
+        "base_vector_path": os.path.join(base_dir, "VectorDB")
+    }
+
+def get_pipeline_config(user_id: int) -> Dict[str, Any]:
+    """Get configuration for DiaryIndexingPipeline."""
+    paths = get_user_paths(user_id)
+    
+    return {
+        "db_path": paths["diary_db_path"],
+        "persist_directory": paths["vector_db_path"],
+        "collection_name": f"user_{user_id}_diary_entries",
+        "google_api_key": os.getenv("GOOGLE_API_KEY"),
+        "chunk_size": 800,
+        "chunk_overlap": 100,
+        "batch_size": 50,
+        "user_id": user_id
+    }
+
+def check_vector_db_exists(user_id: int) -> bool:
+    """Check if vector database exists for user."""
+    paths = get_user_paths(user_id)
+    return os.path.exists(paths["vector_db_path"])
+
+def get_document_count(user_id: int) -> int:
+    """Get document count from vector database."""
+    try:
+        if user_id in rag_systems_cache:
+            return rag_systems_cache[user_id].get_document_count()
+        
+        if not check_vector_db_exists(user_id):
+            return 0
+        
+        # Create temporary RAG system to check count
+        paths = get_user_paths(user_id)
+        temp_rag = create_rag_system(
+            user_id=user_id,
+            base_vector_path=paths["base_vector_path"],
+            google_api_key=os.getenv("GOOGLE_API_KEY")
+        )
+        
+        if temp_rag:
+            return temp_rag.get_document_count()
+        return 0
+        
+    except Exception as e:
+        logger.error(f"Error getting document count for user {user_id}: {e}")
+        return 0
+
+def get_or_create_rag_system(user_id: int) -> DiaryRAGSystem:
+    """Get existing RAG system or create new one."""
+    if user_id not in rag_systems_cache:
+        if not check_vector_db_exists(user_id):
+            raise HTTPException(
+                status_code=404,
+                detail=f"Vector database not found for user {user_id}. Please run indexing first."
+            )
+        
+        paths = get_user_paths(user_id)
+        rag_system = create_rag_system(
+            user_id=user_id,
+            base_vector_path=paths["base_vector_path"],
+            google_api_key=os.getenv("GOOGLE_API_KEY")
+        )
+        
+        if not rag_system:
+            raise HTTPException(
+                status_code=500,
+                detail=f"Failed to create RAG system for user {user_id}"
+            )
+        
+        rag_systems_cache[user_id] = rag_system
+        logger.info(f"Created RAG system for user {user_id}")
+    
+    return rag_systems_cache[user_id]
+
+# ========================================
+# API ENDPOINTS
+# ========================================
+
+@app.get("/")
+async def root():
+    """Health check endpoint."""
+    return {
+        "message": "Personal Diary RAG Service is running",
+        "version": "1.0.0",
+        "cached_users": list(rag_systems_cache.keys()),
+        "vector_db_base": os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "VectorDB")
+    }
+
+@app.get("/health")
+async def health_check():
+    """Detailed health check."""
+    try:
+        google_api_key = os.getenv("GOOGLE_API_KEY")
+        base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        vector_db_base = os.path.join(base_dir, "VectorDB")
+        
+        return {
+            "status": "healthy",
+            "google_api_configured": bool(google_api_key),
+            "vector_db_base_exists": os.path.exists(vector_db_base),
+            "cached_users": list(rag_systems_cache.keys()),
+            "timestamp": datetime.now().isoformat()
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Health check failed: {str(e)}")
+
+@app.get("/users/{user_id}/ai-availability")
+async def check_ai_availability(user_id: int):
+    """Check AI availability and provide detailed status for troubleshooting."""
+    try:
+        # Check all prerequisites for AI availability
+        availability_info = {
+            "user_id": user_id,
+            "overall_status": "checking",
+            "checks": {
+                "rag_modules": {
+                    "available": RAG_MODULES_AVAILABLE,
+                    "status": "✅ Available" if RAG_MODULES_AVAILABLE else "❌ Not Available",
+                    "details": "Required modules: DiaryIndexingPipeline, DiaryRAGSystem"
+                },
+                "google_api_key": {
+                    "configured": bool(os.getenv("GOOGLE_API_KEY")),
+                    "status": "✅ Configured" if os.getenv("GOOGLE_API_KEY") else "❌ Not Configured",
+                    "details": "Required for embeddings and LLM responses"
+                },
+                "vector_database": {
+                    "exists": check_vector_db_exists(user_id),
+                    "status": "✅ Exists" if check_vector_db_exists(user_id) else "⚠️ Not Found",
+                    "path": get_user_paths(user_id)["vector_db_path"]
+                },
+                "document_count": {
+                    "count": get_document_count(user_id),
+                    "status": "✅ Has Documents" if get_document_count(user_id) > 0 else "⚠️ Empty",
+                    "details": f"{get_document_count(user_id)} documents indexed"
+                }
+            },
+            "recommendations": [],
+            "actions": []
+        }
+        
+        # Determine overall status and recommendations
+        if not RAG_MODULES_AVAILABLE:
+            availability_info["overall_status"] = "unavailable"
+            availability_info["recommendations"].append("Install missing RAG modules")
+            availability_info["actions"].append({
+                "action": "check_imports",
+                "description": "Verify DiaryIndexingPipeline and DiaryRAGSystem imports"
+            })
+        elif not os.getenv("GOOGLE_API_KEY"):
+            availability_info["overall_status"] = "not_configured"
+            availability_info["recommendations"].append("Configure Google API key")
+            availability_info["actions"].append({
+                "action": "set_api_key",
+                "description": "Add GOOGLE_API_KEY to environment variables"
+            })
+        elif not check_vector_db_exists(user_id):
+            availability_info["overall_status"] = "needs_indexing"
+            availability_info["recommendations"].append("Create vector database for user")
+            availability_info["actions"].append({
+                "action": "initial_index",
+                "endpoint": f"/users/{user_id}/auto-index-new-entry",
+                "description": "Run initial indexing to create vector database"
+            })
+        elif get_document_count(user_id) == 0:
+            availability_info["overall_status"] = "empty_database"
+            availability_info["recommendations"].append("Add diary entries or rebuild index")
+            availability_info["actions"].append({
+                "action": "check_diary_entries",
+                "description": "Verify user has diary entries in database"
+            })
+            availability_info["actions"].append({
+                "action": "rebuild_index",
+                "endpoint": f"/users/{user_id}/auto-index-new-entry",
+                "description": "Rebuild vector database from existing entries"
+            })
+        else:
+            availability_info["overall_status"] = "available"
+            availability_info["recommendations"].append("AI is ready for use")
+            availability_info["actions"].append({
+                "action": "query_ready",
+                "endpoint": f"/users/{user_id}/query",
+                "description": "AI is ready to answer questions"
+            })
+        
+        # Add cache status
+        availability_info["cache_status"] = {
+            "user_cached": user_id in rag_systems_cache,
+            "total_cached_users": len(rag_systems_cache),
+            "cached_users": list(rag_systems_cache.keys())
+        }
+        
+        return availability_info
+        
+    except Exception as e:
+        logger.error(f"Error checking AI availability for user {user_id}: {e}")
+        return {
+            "user_id": user_id,
+            "overall_status": "error",
+            "error": str(e),
+            "recommendations": ["Check service logs for detailed error information"],
+            "actions": [{
+                "action": "check_logs",
+                "description": "Review service logs for error details"
+            }]
+        }
+
+@app.post("/users/{user_id}/fix-ai-availability")
+async def fix_ai_availability(user_id: int):
+    """Attempt to automatically fix AI availability issues."""
+    try:
+        if not RAG_MODULES_AVAILABLE:
+            return {
+                "status": "cannot_fix",
+                "reason": "RAG modules not available - requires code/environment fix",
+                "action_needed": "Install missing Python modules"
+            }
+        
+        if not os.getenv("GOOGLE_API_KEY"):
+            return {
+                "status": "cannot_fix", 
+                "reason": "Google API key not configured",
+                "action_needed": "Set GOOGLE_API_KEY environment variable"
+            }
+        
+        # Try to fix vector database issues
+        if not check_vector_db_exists(user_id) or get_document_count(user_id) == 0:
+            logger.info(f"Attempting to fix AI availability for user {user_id}")
+            
+            # Clear cache first
+            if user_id in rag_systems_cache:
+                del rag_systems_cache[user_id]
+            
+            # Create/rebuild vector database
+            config = get_pipeline_config(user_id)
+            paths = get_user_paths(user_id)
+            os.makedirs(os.path.dirname(paths["vector_db_path"]), exist_ok=True)
+            
+            pipeline = DiaryIndexingPipeline(**config)
+            results = pipeline.run_full_pipeline(clear_existing=True)
+            
+            if results.get('status') == 'completed_successfully':
+                doc_count = get_document_count(user_id)
+                return {
+                    "status": "fixed",
+                    "action_taken": "Created/rebuilt vector database",
+                    "documents_processed": results.get('documents_loaded', 0),
+                    "chunks_created": results.get('chunks_created', 0),
+                    "final_document_count": doc_count,
+                    "ai_status": "ready" if doc_count > 0 else "empty"
+                }
+            else:
+                return {
+                    "status": "fix_failed",
+                    "reason": "Failed to create vector database",
+                    "error": format_error_message(results.get('errors', 'Unknown error'))
+                }
+        else:
+            return {
+                "status": "already_available",
+                "message": "AI is already available for this user",
+                "document_count": get_document_count(user_id)
+            }
+            
+    except Exception as e:
+        logger.error(f"Error fixing AI availability for user {user_id}: {e}")
+        return {
+            "status": "error",
+            "error": str(e),
+            "action_needed": "Check service logs and try manual troubleshooting"
+        }
+
+@app.get("/users/{user_id}/status", response_model=UserStatusResponse)
+async def get_user_status(user_id: int):
+    """Get RAG system status for a user."""
+    try:
+        paths = get_user_paths(user_id)
+        
+        if not check_vector_db_exists(user_id):
+            return UserStatusResponse(
+                user_id=user_id,
+                status="not_indexed",
+                document_count=0,
+                vector_db_path=paths["vector_db_path"]
+            )
+        
+        doc_count = get_document_count(user_id)
+        
+        return UserStatusResponse(
+            user_id=user_id,
+            status="ready" if doc_count > 0 else "empty",
+            document_count=doc_count,
+            vector_db_path=paths["vector_db_path"],
+            last_updated=datetime.now().isoformat()
+        )
+        
+    except Exception as e:
+        logger.error(f"Error getting status for user {user_id}: {e}")
+        return UserStatusResponse(
+            user_id=user_id,
+            status="error",
+            document_count=0,
+            vector_db_path="",
+            error=str(e)
+        )
+
+@app.post("/users/{user_id}/index", response_model=IndexResponse)
+async def index_user_data(user_id: int, request: IndexRequest, background_tasks: BackgroundTasks):
+    """Index diary entries for a user."""
+    start_time = datetime.now()
+    
+    try:
+        # Ensure VectorDB directory exists
+        paths = get_user_paths(user_id)
+        os.makedirs(os.path.dirname(paths["vector_db_path"]), exist_ok=True)
+        
+        # Get pipeline configuration
+        config = get_pipeline_config(user_id)
+        
+        logger.info(f"Starting indexing for user {user_id} with config: {config}")
+        
+        # Create and run pipeline
+        pipeline = DiaryIndexingPipeline(**config)
+        
+        if request.start_date and request.end_date:
+            # Date range indexing
+            results = pipeline.run_full_pipeline(
+                start_date=request.start_date,
+                end_date=request.end_date,
+                clear_existing=request.clear_existing
+            )
+        else:
+            # Full indexing
+            results = pipeline.run_full_pipeline(clear_existing=request.clear_existing)
+        
+        processing_time = (datetime.now() - start_time).total_seconds()
+        
+        if results.get('status') == 'completed_successfully':
+            # Clear cache to force reload
+            if user_id in rag_systems_cache:
+                del rag_systems_cache[user_id]
+            
+            return IndexResponse(
+                user_id=user_id,
+                status="success",
+                documents_processed=results.get('documents_loaded', 0),
+                chunks_created=results.get('chunks_created', 0),
+                vector_db_path=paths["vector_db_path"],
+                processing_time=processing_time
+            )
+        else:
+            return IndexResponse(
+                user_id=user_id,
+                status="failed",
+                documents_processed=0,
+                chunks_created=0,
+                vector_db_path=paths["vector_db_path"],
+                processing_time=processing_time,
+                error=format_error_message(results.get('errors', 'Unknown error'))
+            )
+            
+    except Exception as e:
+        processing_time = (datetime.now() - start_time).total_seconds()
+        logger.error(f"Indexing error for user {user_id}: {e}")
+        
+        return IndexResponse(
+            user_id=user_id,
+            status="error",
+            documents_processed=0,
+            chunks_created=0,
+            vector_db_path="",
+            processing_time=processing_time,
+            error=str(e)
+        )
+
+@app.post("/users/{user_id}/incremental-index")
+async def incremental_index(user_id: int, start_date: str = None):
+    """Run incremental indexing for user."""
+    try:
+        config = get_pipeline_config(user_id)
+        pipeline = DiaryIndexingPipeline(**config)
+        
+        if start_date:
+            results = pipeline.incremental_update(start_date)
+        else:
+            # Default to last 7 days
+            from datetime import timedelta
+            default_start = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d")
+            results = pipeline.incremental_update(default_start)
+        
+        if results.get('status') == 'success':
+            # Clear cache to force reload
+            if user_id in rag_systems_cache:
+                del rag_systems_cache[user_id]
+            
+            return {
+                "user_id": user_id,
+                "status": "success",
+                "documents_added": results.get('documents_added', 0),
+                "start_date": start_date or default_start
+            }
+        else:
+            raise HTTPException(
+                status_code=500,
+                detail=f"Incremental indexing failed: {results.get('error', 'Unknown error')}"
+            )
+            
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Incremental indexing error for user {user_id}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/users/{user_id}/query", response_model=QueryResponse)
+async def query_user_rag(
+    user_id: int,
+    query: str = Query(...),
+    fast_mode: bool = Query(False),
+    chat_history: str = Query("[]")
+):
+    """Query RAG system for a user."""
+    start_time = datetime.now()
+    import json
+
+    try:
+        rag_system = get_or_create_rag_system(user_id)
+        chat_history_list = json.loads(chat_history)
+        if fast_mode:
+            response = rag_system.generate_fast_response(query=query)
+        else:
+            response = rag_system.generate_contextual_response(
+                query=query,
+                chat_history=chat_history_list
+            )
+        processing_time = (datetime.now() - start_time).total_seconds()
+        return QueryResponse(
+            user_id=user_id,
+            response=response,
+            processing_time=processing_time,
+            documents_used=5,
+            fast_mode=fast_mode
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Query error for user {user_id}: {e}")
+        raise HTTPException(status_code=500, detail=f"Query failed: {str(e)}")
+
+@app.post("/users/{user_id}/auto-index-new-entry")
+async def auto_index_new_entry(user_id: int):
+    """Auto-index after saving new diary entry. Creates initial index if not exists."""
+    try:
+        if not RAG_MODULES_AVAILABLE:
+            return {"status": "skipped", "reason": "RAG modules not available"}
+        
+        # Check if vector DB exists
+        if not check_vector_db_exists(user_id):
+            # First time - create full index
+            logger.info(f"Creating initial vector database for user {user_id}")
+            
+            config = get_pipeline_config(user_id)
+            paths = get_user_paths(user_id)
+            os.makedirs(os.path.dirname(paths["vector_db_path"]), exist_ok=True)
+            
+            pipeline = DiaryIndexingPipeline(**config)
+            results = pipeline.run_full_pipeline(clear_existing=True)
+            
+            if results.get('status') == 'completed_successfully':
+                # Clear cache to force reload
+                if user_id in rag_systems_cache:
+                    del rag_systems_cache[user_id]
+                
+                return {
+                    "status": "initial_index_created",
+                    "message": f"Created initial vector database for user {user_id}",
+                    "documents_processed": results.get('documents_loaded', 0),
+                    "chunks_created": results.get('chunks_created', 0)
+                }
+            else:
+                return {
+                    "status": "failed",
+                    "error": format_error_message(results.get('errors', 'Unknown error'))
+                }
+        else:
+            # Incremental update for existing DB
+            config = get_pipeline_config(user_id)
+            pipeline = DiaryIndexingPipeline(**config)
+            
+            # Get recent entries (last 3 days to catch new ones)
+            from datetime import timedelta
+            start_date = (datetime.now() - timedelta(days=3)).strftime("%Y-%m-%d")
+            results = pipeline.incremental_update(start_date)
+            
+            if results.get('status') == 'success':
+                # Clear cache to force reload
+                if user_id in rag_systems_cache:
+                    del rag_systems_cache[user_id]
+                
+                documents_added = results.get('documents_added', 0)
+                return {
+                    "status": "incremental_update_success",
+                    "message": f"Updated vector database for user {user_id}",
+                    "documents_added": documents_added
+                }
+            else:
+                # If incremental fails, try full rebuild
+                logger.warning(f"Incremental update failed for user {user_id}, trying full rebuild")
+                results = pipeline.run_full_pipeline(clear_existing=True)
+                
+                if results.get('status') == 'completed_successfully':
+                    if user_id in rag_systems_cache:
+                        del rag_systems_cache[user_id]
+                    
+                    return {
+                        "status": "full_rebuild_success",
+                        "message": f"Rebuilt vector database for user {user_id}",
+                        "documents_processed": results.get('documents_loaded', 0)
+                    }
+                else:
+                    return {
+                        "status": "failed",
+                        "error": f"Both incremental and full rebuild failed: {format_error_message(results.get('errors', 'Unknown error'))}"
+                    }
+                
+    except Exception as e:
+        logger.error(f"Auto-index error for user {user_id}: {e}")
+        return {"status": "error", "error": str(e)}
+
+@app.delete("/users/{user_id}/cache")
+async def clear_user_cache(user_id: int):
+    """Clear RAG system cache for a user."""
+    if user_id in rag_systems_cache:
+        del rag_systems_cache[user_id]
+        logger.info(f"Cleared cache for user {user_id}")
+        return {"message": f"Cache cleared for user {user_id}"}
+    else:
+        return {"message": f"No cache found for user {user_id}"}
+
+@app.delete("/users/{user_id}/vector-db")
+async def delete_user_vector_db(user_id: int):
+    """Delete vector database for a user."""
+    try:
+        paths = get_user_paths(user_id)
+        
+        # Clear cache first
+        if user_id in rag_systems_cache:
+            del rag_systems_cache[user_id]
+        
+        # Delete vector database directory
+        if os.path.exists(paths["vector_db_path"]):
+            import shutil
+            shutil.rmtree(paths["vector_db_path"])
+            logger.info(f"Deleted vector database for user {user_id}")
+            return {"message": f"Vector database deleted for user {user_id}"}
+        else:
+            return {"message": f"No vector database found for user {user_id}"}
+            
+    except Exception as e:
+        logger.error(f"Error deleting vector database for user {user_id}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/stats")
+async def get_service_stats():
+    """Get service statistics."""
+    try:
+        base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        vector_db_base = os.path.join(base_dir, "VectorDB")
+        
+        # Get list of existing vector databases
+        existing_dbs = []
+        if os.path.exists(vector_db_base):
+            for item in os.listdir(vector_db_base):
+                if item.startswith("user_") and item.endswith("_vector_db"):
+                    user_id = int(item.replace("user_", "").replace("_vector_db", ""))
+                    doc_count = get_document_count(user_id)
+                    existing_dbs.append({
+                        "user_id": user_id,
+                        "path": os.path.join(vector_db_base, item),
+                        "document_count": doc_count
+                    })
+        
+        return {
+            "cached_users": list(rag_systems_cache.keys()),
+            "total_cached_systems": len(rag_systems_cache),
+            "existing_vector_databases": existing_dbs,
+            "vector_db_base_path": vector_db_base,
+            "service_status": "running"
+        }
+        
+    except Exception as e:
+        logger.error(f"Error getting stats: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    # Ensure VectorDB directory exists
+    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    vector_db_dir = os.path.join(base_dir, "VectorDB")
+    os.makedirs(vector_db_dir, exist_ok=True)
+    
+    print(f"🚀 Starting RAG Service...")
+    print(f"📁 Vector DB base path: {vector_db_dir}")
+    print(f"🔑 Google API Key configured: {bool(os.getenv('GOOGLE_API_KEY'))}")
+    
+    uvicorn.run(app, host="127.0.0.1", port=8001, reload=False)

clean_repo/src/simple_diary_chatbot.py

@@ -0,0 +1,274 @@
+"""
+Simple Diary Chatbot (RAG Minimal Core)
+
+Chức năng bắt buộc (giữ thật đơn giản nhưng vẫn là RAG):
+1. add    -> Lưu entry vào SQLite + CHUNK + EMBEDDING bắt buộc + lưu vector
+2. delete -> Xoá entry (DB + vector theo entry_id)
+3. chat   -> similarity search (k), nếu có API key thì generate câu trả lời, không thì trả về context
+
+Embedding LÀ BẮT BUỘC (sản phẩm RAG). Nếu không có GOOGLE_API_KEY sẽ báo lỗi rõ ràng.
+
+Chunking tối giản: cắt theo độ dài cố định (mặc định 800 ký tự) và không overlap để giảm phức tạp.
+
+File này thay thế các pipeline phức tạp trước đây khi bạn chỉ cần RAG CRUD cơ bản.
+"""
+
+from __future__ import annotations
+
+import os
+import sqlite3
+from dataclasses import dataclass
+from typing import List, Optional, Dict, Any
+from datetime import datetime
+import logging
+from textwrap import wrap
+import asyncio
+
+# Fix event loop issue for Streamlit
+try:
+    import nest_asyncio
+    nest_asyncio.apply()
+except ImportError:
+    pass
+
+# Dùng lại lớp embedding hiện có (Chroma + Google Embedding)
+from Indexingstep.embedding_and_storing import DiaryEmbeddingAndStorage
+
+try:
+    import google.generativeai as genai  # type: ignore
+except Exception:  # pragma: no cover
+    genai = None
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("simple_diary")
+
+DB_PATH = os.path.join(os.getcwd(), "diary.db")  # Một file DB duy nhất
+CHUNK_SIZE = 800  # Có thể chỉnh nếu cần
+
+
+def get_conn():
+    return sqlite3.connect(DB_PATH)
+
+
+def ensure_db():
+    conn = get_conn()
+    cur = conn.cursor()
+    cur.execute(
+        """
+        CREATE TABLE IF NOT EXISTS diary_entries (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            date TEXT NOT NULL,
+            content TEXT NOT NULL,
+            tags TEXT DEFAULT '',
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """
+    )
+    conn.commit()
+    conn.close()
+
+
+@dataclass
+class DiaryEntry:
+    id: int
+    date: str
+    content: str
+    tags: str
+    created_at: str
+
+
+class SimpleDiaryChatbot:
+    """Core RAG tối giản – luôn yêu cầu embedding hoạt động."""
+
+    def __init__(self, api_key: Optional[str] = None, user_id: int = 1, chunk_size: int = CHUNK_SIZE):
+        if api_key:
+            os.environ["GOOGLE_API_KEY"] = api_key
+        ensure_db()
+        self.user_id = user_id
+        self.chunk_size = chunk_size
+
+        key = os.getenv("GOOGLE_API_KEY")
+        if not key:
+            raise RuntimeError(
+                "GOOGLE_API_KEY chưa được thiết lập. Set trong PowerShell: $env:GOOGLE_API_KEY='YOUR_KEY'"
+            )
+
+        # Fix event loop for Streamlit
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+        # Khởi tạo embedding + vector store (bắt buộc)
+        self.embedding_store = DiaryEmbeddingAndStorage(user_id=user_id, api_key=key)
+
+        # (Tuỳ chọn) LLM để tạo câu trả lời tự nhiên – nếu lỗi vẫn tiếp tục dùng context
+        self._model = None
+        if genai:
+            try:
+                genai.configure(api_key=key)
+                self._model = genai.GenerativeModel("gemini-1.5-flash")
+            except Exception as e:
+                logger.warning(f"Không khởi tạo được LLM (tiếp tục với retrieval-only): {e}")
+
+    # ------------- CRUD -------------
+    def _chunk(self, text: str) -> List[str]:
+        """Chunk đơn giản theo độ dài cố định, cắt ở khoảng trắng gần nhất nếu có."""
+        if len(text) <= self.chunk_size:
+            return [text]
+        chunks: List[str] = []
+        start = 0
+        while start < len(text):
+            end = min(start + self.chunk_size, len(text))
+            # Cố gắng lùi về khoảng trắng để tránh cắt từ
+            if end < len(text):
+                last_space = text.rfind(" ", start, end)
+                if last_space != -1 and last_space - start > self.chunk_size * 0.5:
+                    end = last_space
+            chunks.append(text[start:end].strip())
+            start = end
+        return [c for c in chunks if c]
+
+    def add_entry(self, date: str, content: str, tags: str = "") -> int:
+        conn = get_conn()
+        cur = conn.cursor()
+        cur.execute(
+            "INSERT INTO diary_entries(date, content, tags) VALUES (?, ?, ?)",
+            (date, content, tags),
+        )
+        entry_id = cur.lastrowid
+        conn.commit()
+        conn.close()
+
+        # Chunk + embed từng chunk (metadata chung entry)
+        chunks = self._chunk(content)
+        metadatas = [
+            {"entry_id": entry_id, "date": date, "tags": tags, "chunk_index": i, "total_chunks": len(chunks)}
+            for i, _ in enumerate(chunks)
+        ]
+        self.embedding_store.embed_and_store_texts(chunks, metadatas)
+        # logger.info(f"Added entry {entry_id} với {len(chunks)} chunk")
+        return entry_id
+
+    def delete_entry(self, entry_id: int) -> bool:
+        # Xoá vector theo metadata
+        try:
+            self.embedding_store.delete_documents_by_metadata({"entry_id": entry_id})
+        except Exception as e:
+            logger.warning(f"Failed to delete vectors for entry {entry_id}: {e}")
+
+        conn = get_conn()
+        cur = conn.cursor()
+        cur.execute("DELETE FROM diary_entries WHERE id = ?", (entry_id,))
+        deleted = cur.rowcount
+        conn.commit()
+        conn.close()
+        if deleted:
+            logger.info(f"Deleted entry {entry_id}")
+            return True
+        logger.warning(f"Entry {entry_id} not found")
+        return False
+
+    def list_entries(self, limit: int = 10) -> List[DiaryEntry]:
+        conn = get_conn()
+        cur = conn.cursor()
+        cur.execute(
+            "SELECT id, date, content, tags, created_at FROM diary_entries ORDER BY created_at DESC LIMIT ?",
+            (limit,),
+        )
+        rows = [DiaryEntry(*r) for r in cur.fetchall()]
+        conn.close()
+        return rows
+
+    # ------------- Chat -------------
+    def chat(self, question: str, k: int = 4) -> Dict[str, Any]:
+        """
+        Trả về:
+            {
+              'answer': str,
+              'contexts': [ { 'snippet': ..., 'date': ..., 'entry_id': ... } ]
+            }
+        """
+        try:
+            results = self.embedding_store.similarity_search(question, k=k)
+        except Exception as e:
+            logger.warning(f"Similarity search failed: {e}")
+            results = []
+        contexts = []
+        for doc in results:
+            contexts.append(
+                {
+                    "snippet": doc.page_content[:300],
+                    "date": doc.metadata.get("date"),
+                    "entry_id": doc.metadata.get("entry_id"),
+                    "tags": doc.metadata.get("tags"),
+                }
+            )
+
+        if self._model and contexts:
+            context_text = "\n".join(
+                [f"[Entry {c['entry_id']} - {c['date']}] {c['snippet']}" for c in contexts]
+            )
+            prompt = (
+                "You are a helpful diary assistant. Use only the context below to answer.\n\n"
+                f"CONTEXT:\n{context_text}\n\nQUESTION: {question}\n\nAnswer in the same language as the question."
+            )
+            try:
+                resp = self._model.generate_content(prompt)
+                answer = resp.text.strip()
+            except Exception as e:
+                answer = f"(LLM error, showing raw context) -> {e}\n" + " | ".join(
+                    c["snippet"] for c in contexts
+                )
+        else:
+            answer = " | ".join(c["snippet"] for c in contexts) if contexts else "Không tìm thấy nội dung liên quan."
+
+        return {"answer": answer, "contexts": contexts}
+
+
+def _cli():  # Simple command line interface
+    import argparse
+    parser = argparse.ArgumentParser(description="Simple Diary Chatbot")
+    sub = parser.add_subparsers(dest="cmd")
+
+    p_add = sub.add_parser("add", help="Add a diary entry")
+    p_add.add_argument("--date", default=datetime.now().strftime("%Y-%m-%d"))
+    p_add.add_argument("--content", required=True)
+    p_add.add_argument("--tags", default="")
+
+    p_del = sub.add_parser("delete", help="Delete an entry by id")
+    p_del.add_argument("--id", type=int, required=True)
+
+    p_chat = sub.add_parser("chat", help="Ask a question")
+    p_chat.add_argument("--q", required=True, help="Question")
+    p_chat.add_argument("--k", type=int, default=4)
+
+    p_list = sub.add_parser("list", help="List recent entries")
+    p_list.add_argument("--limit", type=int, default=5)
+
+    args = parser.parse_args()
+    bot = SimpleDiaryChatbot(api_key=os.getenv("GOOGLE_API_KEY"))
+
+    if args.cmd == "add":
+        eid = bot.add_entry(args.date, args.content, args.tags)
+        print(f"Added entry id={eid}")
+    elif args.cmd == "delete":
+        ok = bot.delete_entry(args.id)
+        print("Deleted" if ok else "Not found")
+    elif args.cmd == "chat":
+        resp = bot.chat(args.q, k=args.k)
+        print("Answer:\n", resp["answer"]) 
+        print("\nContexts:")
+        for c in resp["contexts"]:
+            print(f"- ({c['entry_id']}) {c['date']} :: {c['snippet'][:80]}...")
+    elif args.cmd == "list":
+        entries = bot.list_entries(limit=args.limit)
+        for e in entries:
+            print(f"{e.id} | {e.date} | {e.tags} | {e.content[:60]}...")
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    _cli()

clean_repo/src/streamlit_app.py

@@ -0,0 +1,40 @@
+import altair as alt
+import numpy as np
+import pandas as pd
+import streamlit as st
+
+"""
+# Welcome to Streamlit!
+
+Edit `/streamlit_app.py` to customize this app to your heart's desire :heart:.
+If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
+forums](https://discuss.streamlit.io).
+
+In the meantime, below is an example of what you can do with just a few lines of code:
+"""
+
+num_points = st.slider("Number of points in spiral", 1, 10000, 1100)
+num_turns = st.slider("Number of turns in spiral", 1, 300, 31)
+
+indices = np.linspace(0, 1, num_points)
+theta = 2 * np.pi * num_turns * indices
+radius = indices
+
+x = radius * np.cos(theta)
+y = radius * np.sin(theta)
+
+df = pd.DataFrame({
+    "x": x,
+    "y": y,
+    "idx": indices,
+    "rand": np.random.randn(num_points),
+})
+
+st.altair_chart(alt.Chart(df, height=700, width=700)
+    .mark_point(filled=True)
+    .encode(
+        x=alt.X("x", axis=None),
+        y=alt.Y("y", axis=None),
+        color=alt.Color("idx", legend=None, scale=alt.Scale()),
+        size=alt.Size("rand", legend=None, scale=alt.Scale(range=[1, 150])),
+    ))