Refactor blog data utilities and configuration

- Removed the Jupyter notebook `utils_data_loading.ipynb` and migrated utility functions to `blog_utils.py`.
- Created a new configuration file `config.py` to manage environment variables and default settings.
- Implemented a script `update_blog_data.py` for updating the blog data vector store with command-line options.
- Added a JSON file `blog_stats_20250510_161540.json` to store blog statistics.
- Enhanced document processing functions for better modularity and error handling.

BLOG_DATA_UTILS.md

@@ -4,24 +4,30 @@ This directory contains utilities for loading, processing, and maintaining blog
 
 ## Available Tools
 
-### `utils_data_loading.ipynb`
+### `blog_utils.py`
 
-This notebook contains utility functions for:
+This Python module contains utility functions for:
 - Loading blog posts from the data directory
 - Processing and enriching metadata (adding URLs, titles, etc.)
 - Getting statistics about the documents
 - Creating and updating vector embeddings
 - Loading existing vector stores
 
-### `update_blog_data.ipynb`
+### `update_blog_data.py`
 
-This notebook demonstrates how to:
-- Use the utility functions to update the blog data
+This script allows you to:
+- Update the blog data when new posts are published
 - Process new blog posts
 - Update the vector store
-- Test the updated system with sample queries
 - Track changes over time
 
+### Legacy Notebooks (Reference Only)
+
+The following notebooks are kept for reference but the functionality has been moved to Python modules:
+
+- `utils_data_loading.ipynb` - Contains the original utility functions
+- `update_blog_data.ipynb` - Demonstrates the update workflow
+
 ## How to Use
 
 ### Updating Blog Data
@@ -29,12 +35,17 @@ This notebook demonstrates how to:
 When new blog posts are published, follow these steps:
 
 1. Add the markdown files to the `data/` directory
-2. Run the update notebook:
+2. Run the update script:
    ```bash
    cd /home/mafzaal/source/lets-talk
-   uv run jupyter nbconvert --to notebook --execute update_blog_data.ipynb --output executed_update_$(date +%Y%m%d).ipynb
+   uv run python update_blog_data.py
    ```
    
+   You can also force recreation of the vector store:
+   ```bash
+   uv run python update_blog_data.py --force-recreate
+   ```
+
 This will:
 - Load all blog posts (including new ones)
 - Update the vector embeddings
@@ -50,23 +61,22 @@ VECTOR_STORAGE_PATH=./db/vectorstore_v3    # Path to vector store
 EMBEDDING_MODEL=Snowflake/snowflake-arctic-embed-l  # Embedding model
 QDRANT_COLLECTION=thedataguy_documents     # Collection name
 BLOG_BASE_URL=https://thedataguy.pro/blog/ # Base URL for blog
-FORCE_RECREATE_EMBEDDINGS=false           # Whether to force recreation
 ```
 
 ### In the Chainlit App
 
-The Chainlit app (`app.py`) has been updated to use these utility functions if available. It falls back to direct initialization if they can't be loaded.
+The Chainlit app (`app.py`) has been updated to use these utility functions from the `blog_utils.py` module. It falls back to notebook import and direct initialization if there are any issues.
 
 ## Adding Custom Processing
 
 To add custom processing for blog posts:
 
-1. Edit the `update_document_metadata` function in `utils_data_loading.ipynb`
+1. Edit the `update_document_metadata` function in `blog_utils.py`
 2. Add any additional enrichment or processing steps
-3. Update the vector store using the `update_blog_data.ipynb` notebook
+3. Update the vector store using the `update_blog_data.py` script
 
 ## Future Improvements
 
-- Add support for incremental updates (only process new posts)
-- Add webhook support to automatically update when new posts are published
+- Add scheduled update process for automatically including new blog posts
 - Add tracking of embedding models and versions
+- Add webhook support to automatically update when new posts are published

app.py

@@ -1,9 +1,9 @@
 import os
 import getpass
 import sys
-import importlib.util
 from pathlib import Path
 from operator import itemgetter
+from config import LLM_MODEL, LLM_TEMPERATURE
 from dotenv import load_dotenv
 
 # Load environment variables from .env file
@@ -17,78 +17,17 @@ from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_qdrant import QdrantVectorStore
 from qdrant_client import QdrantClient
 from qdrant_client.http.models import Distance, VectorParams
-
-# Import utility functions from the notebook
-def import_notebook_functions(notebook_path):
-    """Import functions from a Jupyter notebook"""
-    import nbformat
-    from importlib.util import spec_from_loader, module_from_spec
-    from IPython.core.interactiveshell import InteractiveShell
-    
-    # Create a module
-    module_name = Path(notebook_path).stem
-    spec = spec_from_loader(module_name, loader=None)
-    module = module_from_spec(spec)
-    sys.modules[module_name] = module
-    
-    # Read the notebook
-    with open(notebook_path) as f:
-        nb = nbformat.read(f, as_version=4)
-    
-    # Execute code cells
-    shell = InteractiveShell.instance()
-    for cell in nb.cells:
-        if cell.cell_type == 'code':
-            # Skip example code
-            if 'if __name__ == "__main__":' in cell.source:
-                continue
-                
-            code = shell.input_transformer_manager.transform_cell(cell.source)
-            exec(code, module.__dict__)
+import blog_utils
     
-    return module
-
-# Try to import utility functions if available
-try:
-    utils = import_notebook_functions('utils_data_loading.ipynb')
-    
-    # Load vector store using the utility function
-    vector_store = utils.load_vector_store(
-        storage_path=os.environ.get("VECTOR_STORAGE_PATH", "./db/vectorstore_v3"),
-        collection_name=os.environ.get("QDRANT_COLLECTION", "thedataguy_documents"),
-        embedding_model=os.environ.get("EMBEDDING_MODEL", "Snowflake/snowflake-arctic-embed-l")
-    )
-    
-    print("Successfully loaded vector store using utility functions")
-    
-except Exception as e:
-    print(f"Could not load utility functions: {e}")
-    print("Falling back to direct initialization")
-    
-    # Get vector storage path from .env file with fallback
-    storage_path = Path(os.environ.get("VECTOR_STORAGE_PATH", "./db/vectorstore_v3"))
-    
-    # Load embedding model from environment variable with fallback
-    embedding_model = os.environ.get("EMBEDDING_MODEL", "Snowflake/snowflake-arctic-embed-l")
-    huggingface_embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
-    
-    # Set up Qdrant vectorstore from existing collection
-    collection_name = os.environ.get("QDRANT_COLLECTION", "thedataguy_documents")
-    
-    vector_store = QdrantVectorStore.from_existing_collection(
-        path=storage_path,
-        collection_name=collection_name,
-        embedding=huggingface_embeddings,
-    )
-
+# Load vector store using the utility function
+vector_store = blog_utils.load_vector_store()
 
 # Create a retriever
 retriever = vector_store.as_retriever()
 
 # Set up ChatOpenAI with environment variables
-llm_model = os.environ.get("LLM_MODEL", "gpt-4o-mini")
-temperature = float(os.environ.get("TEMPERATURE", "0"))
-llm = ChatOpenAI(model=llm_model, temperature=temperature)
+
+llm = ChatOpenAI(model=LLM_MODEL, temperature=LLM_TEMPERATURE)
 
 # Create RAG prompt template
 rag_prompt_template = """\
@@ -149,24 +88,24 @@ async def on_message(message: cl.Message):
     response = chain.invoke({"question": message.content})
     
     # Get the sources to display them
-    sources = []
-    for doc in response["context"]:
-        if "url" in doc.metadata:
-            # Get title from post_title metadata if available, otherwise derive from URL
-            title = doc.metadata.get("post_title", "")
-            if not title:
-                title = doc.metadata["url"].split("/")[-2].replace("-", " ").title()
+    # sources = []
+    # for doc in response["context"]:
+    #     if "url" in doc.metadata:
+    #         # Get title from post_title metadata if available, otherwise derive from URL
+    #         title = doc.metadata.get("post_title", "")
+    #         if not title:
+    #             title = doc.metadata["url"].split("/")[-2].replace("-", " ").title()
                 
-            sources.append(
-                cl.Source(
-                    url=doc.metadata["url"],
-                    title=title
-                )
-            )
+    #         sources.append(
+    #             cl.Source(
+    #                 url=doc.metadata["url"],
+    #                 title=title
+    #             )
+    #         )
 
     # Send the response with sources
     await cl.Message(
         content=response["response"].content,
-        sources=sources
+        #sources=sources
     ).send()
 

blog_utils.py

@@ -0,0 +1,305 @@
+"""
+Blog Data Utilities Module
+
+This module contains utility functions for loading, processing, and storing blog posts
+for the RAG system. It includes functions for loading blog posts from the data directory,
+processing their metadata, and creating vector embeddings.
+"""
+
+import os
+import json
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+
+from langchain_community.document_loaders import DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.schema.document import Document
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_qdrant import QdrantVectorStore
+from qdrant_client import QdrantClient
+
+
+from config import (
+    DATA_DIR,
+    VECTOR_STORAGE_PATH,
+    EMBEDDING_MODEL,
+    QDRANT_COLLECTION,
+    BLOG_BASE_URL
+)
+
+def load_blog_posts(data_dir: str = DATA_DIR, 
+                   glob_pattern: str = "*.md", 
+                   recursive: bool = True, 
+                   show_progress: bool = True) -> List[Document]:
+    """
+    Load blog posts from the specified directory.
+    
+    Args:
+        data_dir: Directory containing the blog posts
+        glob_pattern: Pattern to match files
+        recursive: Whether to search subdirectories
+        show_progress: Whether to show a progress bar
+        
+    Returns:
+        List of Document objects containing the blog posts
+    """
+    text_loader = DirectoryLoader(
+        data_dir, 
+        glob=glob_pattern, 
+        show_progress=show_progress,
+        recursive=recursive
+    )
+    
+    documents = text_loader.load()
+    print(f"Loaded {len(documents)} documents from {data_dir}")
+    return documents
+
+
+def update_document_metadata(documents: List[Document], 
+                           data_dir_prefix: str = DATA_DIR,
+                           blog_base_url: str = BLOG_BASE_URL,
+                           remove_suffix: str = "index.md") -> List[Document]:
+    """
+    Update the metadata of documents to include URL and other information.
+    
+    Args:
+        documents: List of Document objects to update
+        data_dir_prefix: Prefix to replace in source paths
+        blog_base_url: Base URL for the blog posts
+        remove_suffix: Suffix to remove from paths (like index.md)
+        
+    Returns:
+        Updated list of Document objects
+    """
+    for doc in documents:
+        # Create URL from source path
+        doc.metadata["url"] = doc.metadata["source"].replace(data_dir_prefix, blog_base_url)
+        
+        # Remove index.md or other suffix if present
+        if remove_suffix and doc.metadata["url"].endswith(remove_suffix):
+            doc.metadata["url"] = doc.metadata["url"][:-len(remove_suffix)]
+            
+        # Extract post title from the directory structure
+        path_parts = Path(doc.metadata["source"]).parts
+        if len(path_parts) > 1:
+            # Use the directory name as post_slug
+            doc.metadata["post_slug"] = path_parts[-2]
+            doc.metadata["post_title"] = path_parts[-2].replace("-", " ").title()
+            
+        # Add document length as metadata
+        doc.metadata["content_length"] = len(doc.page_content)
+    
+    return documents
+
+
+def get_document_stats(documents: List[Document]) -> Dict[str, Any]:
+    """
+    Get statistics about the documents.
+    
+    Args:
+        documents: List of Document objects
+        
+    Returns:
+        Dictionary with statistics
+    """
+    stats = {
+        "total_documents": len(documents),
+        "total_characters": sum(len(doc.page_content) for doc in documents),
+        "min_length": min(len(doc.page_content) for doc in documents) if documents else 0,
+        "max_length": max(len(doc.page_content) for doc in documents) if documents else 0,
+        "avg_length": sum(len(doc.page_content) for doc in documents) / len(documents) if documents else 0,
+    }
+    
+    # Create a list of document info for analysis
+    doc_info = []
+    for doc in documents:
+        doc_info.append({
+            "url": doc.metadata.get("url", ""),
+            "source": doc.metadata.get("source", ""),
+            "title": doc.metadata.get("post_title", ""),
+            "text_length": doc.metadata.get("content_length", 0),
+        })
+    
+    stats["documents"] = doc_info
+    return stats
+
+
+def display_document_stats(stats: Dict[str, Any]):
+    """
+    Display document statistics in a readable format.
+    
+    Args:
+        stats: Dictionary with statistics from get_document_stats
+    """
+    print(f"Total Documents: {stats['total_documents']}")
+    print(f"Total Characters: {stats['total_characters']}")
+    print(f"Min Length: {stats['min_length']} characters")
+    print(f"Max Length: {stats['max_length']} characters")
+    print(f"Average Length: {stats['avg_length']:.2f} characters")
+    
+    # For use in notebooks where pandas and display are available:
+    try:
+        import pandas as pd
+        from IPython.display import display
+        if stats["documents"]:
+            df = pd.DataFrame(stats["documents"])
+            display(df)
+    except (ImportError, NameError):
+        # Just print the first 5 documents if not in a notebook environment
+        if stats["documents"]:
+            print("\nFirst 5 documents:")
+            for i, doc in enumerate(stats["documents"][:5]):
+                print(f"{i+1}. {doc['title']} ({doc['url']})")
+
+
+def split_documents(documents: List[Document], 
+                   chunk_size: int = 1000, 
+                   chunk_overlap: int = 200) -> List[Document]:
+    """
+    Split documents into chunks for better embedding and retrieval.
+    
+    Args:
+        documents: List of Document objects to split
+        chunk_size: Size of each chunk in characters
+        chunk_overlap: Overlap between chunks in characters
+        
+    Returns:
+        List of split Document objects
+    """
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        length_function=len,
+    )
+    
+    split_docs = text_splitter.split_documents(documents)
+    print(f"Split {len(documents)} documents into {len(split_docs)} chunks")
+    return split_docs
+
+
+def create_vector_store(documents: List[Document], 
+                       storage_path: str = VECTOR_STORAGE_PATH,
+                       collection_name: str = QDRANT_COLLECTION,
+                       embedding_model: str = EMBEDDING_MODEL,
+                       force_recreate: bool = False) -> Optional[QdrantVectorStore]:
+    
+    """
+    Create a vector store from the documents using Qdrant.
+    Args:
+        documents: List of Document objects to embed
+        storage_path: Path to the vector store
+        collection_name: Name of the collection
+        embedding_model: Name of the embedding model
+        force_recreate: Whether to force recreation of the vector store
+    Returns:
+        QdrantVectorStore vector store or None if creation fails
+    """
+
+    vector_store = QdrantVectorStore.from_documents(
+        documents,
+        embedding=HuggingFaceEmbeddings(model_name=embedding_model),
+        collection_name=collection_name,
+        path=storage_path,
+        force_recreate=force_recreate,
+    )
+    
+    return vector_store
+
+
+def load_vector_store(storage_path: str = VECTOR_STORAGE_PATH,
+                     collection_name: str = QDRANT_COLLECTION,
+                     embedding_model: str = EMBEDDING_MODEL) -> Optional[QdrantVectorStore]:
+    """
+    Load an existing vector store.
+    
+    Args:
+        storage_path: Path to the vector store
+        collection_name: Name of the collection
+        embedding_model: Name of the embedding model
+        
+    Returns:
+        QdrantVectorStore vector store or None if it doesn't exist
+    """
+    # Initialize the embedding model
+    embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
+    
+    # Check if vector store exists
+    if not Path(storage_path).exists():
+        print(f"Vector store not found at {storage_path}")
+        return None
+    
+    try:
+        # Initialize Qdrant client
+        client = QdrantClient(path=storage_path)
+        
+        # Create vector store with the client
+        vector_store = QdrantVectorStore(
+            client=client,
+            collection_name=collection_name,
+            embedding=embeddings,
+        )
+        print(f"Loaded vector store from {storage_path}")
+        return vector_store
+    except Exception as e:
+        print(f"Error loading vector store: {e}")
+        return None
+
+
+def process_blog_posts(data_dir: str = DATA_DIR,
+                      create_embeddings: bool = True,
+                      force_recreate_embeddings: bool = False,
+                      storage_path: str = VECTOR_STORAGE_PATH):
+    """
+    Complete pipeline to process blog posts and optionally create vector embeddings.
+    
+    Args:
+        data_dir: Directory containing the blog posts
+        create_embeddings: Whether to create vector embeddings
+        force_recreate_embeddings: Whether to force recreation of embeddings
+        storage_path: Path to the vector store (not used with in-memory approach)
+        
+    Returns:
+        Dictionary with data and vector store (if created)
+    """
+    # Load documents
+    documents = load_blog_posts(data_dir)
+    
+    # Update metadata
+    documents = update_document_metadata(documents)
+    
+
+    # Get and display stats
+    stats = get_document_stats(documents)
+    display_document_stats(stats)
+    
+    result = {
+        "documents": documents,
+        "stats": stats,
+        "vector_store": None
+    }
+    
+    # Create vector store if requested
+    if create_embeddings:
+        # Using in-memory vector store to avoid pickling issues
+        vector_store = create_vector_store(
+            documents, 
+            force_recreate=force_recreate_embeddings
+        )
+        result["vector_store"] = vector_store
+    
+    return result
+
+
+# Allow script to be run directly if needed
+if __name__ == "__main__":
+    print("Blog Data Utilities Module")
+    print("Available functions:")
+    print("- load_blog_posts()")
+    print("- update_document_metadata()")
+    print("- get_document_stats()")
+    print("- display_document_stats()")
+    print("- split_documents()")
+    print("- create_vector_store()")
+    print("- load_vector_store()")
+    print("- process_blog_posts()")

config.py

@@ -0,0 +1,14 @@
+import os
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Configuration with defaults
+DATA_DIR = os.environ.get("DATA_DIR", "data/")
+VECTOR_STORAGE_PATH = os.environ.get("VECTOR_STORAGE_PATH", "./db/vectorstore_v3")
+EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "Snowflake/snowflake-arctic-embed-l")
+QDRANT_COLLECTION = os.environ.get("QDRANT_COLLECTION", "thedataguy_documents")
+BLOG_BASE_URL = os.environ.get("BLOG_BASE_URL", "https://thedataguy.pro/blog/")
+LLM_MODEL = os.environ.get("LLM_MODEL", "gpt-4o-mini")
+LLM_TEMPERATURE = float(os.environ.get("TEMPERATURE", "0"))

main.py

@@ -1,5 +1,48 @@
+import blog_utils
+from update_blog_data import parse_args, save_stats
+
+
 def main():
-    print("Hello from lets-talk!")
+
+    """Main function to update blog data"""
+    args = parse_args()
+
+    print("=== Blog Data Update ===")
+    print(f"Data directory: {args.data_dir}")
+    print(f"Force recreate: {args.force_recreate}")
+    print("========================")
+
+    # Process blog posts without creating embeddings
+    try:
+        # Load and process documents
+        documents = blog_utils.load_blog_posts(args.data_dir)
+        documents = blog_utils.update_document_metadata(documents)
+
+        # Get stats
+        stats = blog_utils.get_document_stats(documents)
+        blog_utils.display_document_stats(stats)
+
+        # Save stats for tracking
+        stats_file = save_stats(stats)
+
+        # Create a reference file for the vector store
+        if args.force_recreate:
+            print("\nAttempting to save vector store reference file...")
+            blog_utils.create_vector_store(documents, force_recreate=args.force_recreate)
+
+        print("\n=== Update Summary ===")
+        print(f"Processed {stats['total_documents']} documents")
+        print(f"Stats saved to: {stats_file}")
+        print("Note: Vector store creation is currently disabled due to pickling issues.")
+        print("      See VECTOR_STORE_ISSUES.md for more information and possible solutions.")
+        print("=====================")
+
+        return 0
+    except Exception as e:
+        print(f"Error: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
 
 
 if __name__ == "__main__":

stats/blog_stats_20250510_161540.json

@@ -0,0 +1,8 @@
+{
+  "timestamp": "20250510_161540",
+  "total_documents": 14,
+  "total_characters": 106275,
+  "min_length": 1900,
+  "max_length": 13468,
+  "avg_length": 7591.071428571428
+}

update_blog_data.ipynb

@@ -12,7 +12,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "6ec048b4",
    "metadata": {},
    "outputs": [],
@@ -21,139 +21,88 @@
     "import os\n",
     "from pathlib import Path\n",
     "from dotenv import load_dotenv\n",
-    "import importlib.util\n",
-    "\n",
-    "# Load environment variables\n",
-    "load_dotenv()\n",
-    "\n",
-    "# Import utility functions from utils_data_loading.ipynb\n",
-    "# We'll do this by first converting the notebook to a Python module"
+    "import importlib.util\n"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7f01d61f",
+   "cell_type": "markdown",
+   "id": "cc19ab4c",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "# Function to import the utility module\n",
-    "def import_notebook_as_module(notebook_path, module_name=\"utils_module\"):\n",
-    "    \"\"\"\n",
-    "    Import a Jupyter notebook as a Python module.\n",
-    "    \n",
-    "    Args:\n",
-    "        notebook_path: Path to the notebook\n",
-    "        module_name: Name to give the module\n",
-    "        \n",
-    "    Returns:\n",
-    "        The imported module\n",
-    "    \"\"\"\n",
-    "    import nbformat\n",
-    "    from importlib.util import spec_from_loader, module_from_spec\n",
-    "    from IPython.core.interactiveshell import InteractiveShell\n",
-    "    \n",
-    "    shell = InteractiveShell.instance()\n",
-    "    \n",
-    "    with open(notebook_path) as f:\n",
-    "        nb = nbformat.read(f, as_version=4)\n",
-    "    \n",
-    "    # Create a module\n",
-    "    spec = spec_from_loader(module_name, loader=None)\n",
-    "    module = module_from_spec(spec)\n",
-    "    sys.modules[module_name] = module\n",
-    "    \n",
-    "    # Execute only the code cells in the notebook\n",
-    "    for cell in nb.cells:\n",
-    "        if cell.cell_type == 'code':\n",
-    "            # Skip cells that start with certain keywords like \"if __name__ == \"__main__\":\"\n",
-    "            if 'if __name__ == \"__main__\":' in cell.source:\n",
-    "                continue\n",
-    "            \n",
-    "            # Execute the cell and store its content in the module\n",
-    "            code = shell.input_transformer_manager.transform_cell(cell.source)\n",
-    "            exec(code, module.__dict__)\n",
-    "    \n",
-    "    return module"
+    "## Update Blog Data Process\n",
+    "\n",
+    "This process will:\n",
+    "1. Load existing blog posts\n",
+    "2. Process and update metadata\n",
+    "3. Create or update vector embeddings"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "774c1373",
+   "execution_count": 7,
+   "id": "3d56f688",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 14/14 [00:00<00:00, 42.05it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loaded 14 documents from data/\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
    "source": [
-    "# Import the utility functions\n",
-    "utils = import_notebook_as_module('utils_data_loading.ipynb')\n",
+    "import blog_utils\n",
     "\n",
-    "# Now you can access all the functions from the utils module\n",
-    "print(\"Successfully imported utility functions.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "85ae6617",
-   "metadata": {},
-   "source": [
-    "## Configuration\n",
+    "docs = blog_utils.load_blog_posts()\n",
+    "docs = blog_utils.update_document_metadata(docs)\n",
     "\n",
-    "Set up the configuration for data processing."
+    "\n"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "54e9ca48",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Configuration (can be overridden from .env file)\n",
-    "DATA_DIR = os.environ.get(\"DATA_DIR\", \"data/\")\n",
-    "VECTOR_STORAGE_PATH = os.environ.get(\"VECTOR_STORAGE_PATH\", \"./db/vectorstore_v3\")\n",
-    "BLOG_BASE_URL = os.environ.get(\"BLOG_BASE_URL\", \"https://thedataguy.pro/blog/\")\n",
-    "FORCE_RECREATE_EMBEDDINGS = os.environ.get(\"FORCE_RECREATE_EMBEDDINGS\", \"false\").lower() == \"true\"\n",
-    "\n",
-    "print(f\"Data Directory: {DATA_DIR}\")\n",
-    "print(f\"Vector Storage Path: {VECTOR_STORAGE_PATH}\")\n",
-    "print(f\"Blog Base URL: {BLOG_BASE_URL}\")\n",
-    "print(f\"Force Recreate Embeddings: {FORCE_RECREATE_EMBEDDINGS}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "cc19ab4c",
+   "id": "a14c70dc",
    "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Document(metadata={'source': 'data/introduction-to-ragas/index.md', 'url': 'https://thedataguy.pro/blog/introduction-to-ragas/', 'post_slug': 'introduction-to-ragas', 'post_title': 'Introduction To Ragas', 'content_length': 6071}, page_content='title: \"Part 1: Introduction to Ragas: The Essential Evaluation Framework for LLM Applications\" date: 2025-04-26T18:00:00-06:00 layout: blog description: \"Explore the essential evaluation framework for LLM applications with Ragas. Learn how to assess performance, ensure accuracy, and improve reliability in Retrieval-Augmented Generation systems.\" categories: [\"AI\", \"RAG\", \"Evaluation\",\"Ragas\"] coverImage: \"https://images.unsplash.com/photo-1593642634367-d91a135587b5?q=80&w=1770&auto=format&fit=crop&ixlib=rb-4.0.3\" readingTime: 7 published: true\\n\\nAs Large Language Models (LLMs) become fundamental components of modern applications, effectively evaluating their performance becomes increasingly critical. Whether you\\'re building a question-answering system, a document retrieval tool, or a conversational agent, you need reliable metrics to assess how well your application performs. This is where Ragas steps in.\\n\\nWhat is Ragas?\\n\\nRagas is an open-source evaluation framework specifically designed for LLM applications, with particular strengths in Retrieval-Augmented Generation (RAG) systems. Unlike traditional NLP evaluation methods, Ragas provides specialized metrics that address the unique challenges of LLM-powered systems.\\n\\nAt its core, Ragas helps answer crucial questions: - Is my application retrieving the right information? - Are the responses factually accurate and consistent with the retrieved context? - Does the system appropriately address the user\\'s query? - How well does my application handle multi-turn conversations?\\n\\nWhy Evaluate LLM Applications?\\n\\nLLMs are powerful but imperfect. They can hallucinate facts, misinterpret queries, or generate convincing but incorrect responses. For applications where accuracy and reliability matter—like healthcare, finance, or education—proper evaluation is non-negotiable.\\n\\nEvaluation serves several key purposes: - Quality assurance: Identify and fix issues before they reach users - Performance tracking: Monitor how changes impact system performance - Benchmarking: Compare different approaches objectively - Continuous improvement: Build feedback loops to enhance your application\\n\\nKey Features of Ragas\\n\\n🎯 Specialized Metrics\\n\\nRagas offers both LLM-based and computational metrics tailored to evaluate different aspects of LLM applications:\\n\\nFaithfulness: Measures if the response is factually consistent with the retrieved context\\n\\nContext Relevancy: Evaluates if the retrieved information is relevant to the query\\n\\nAnswer Relevancy: Assesses if the response addresses the user\\'s question\\n\\nTopic Adherence: Gauges how well multi-turn conversations stay on topic\\n\\n🧪 Test Data Generation\\n\\nCreating high-quality test data is often a bottleneck in evaluation. Ragas helps you generate comprehensive test datasets automatically, saving time and ensuring thorough coverage.\\n\\n🔗 Seamless Integrations\\n\\nRagas works with popular LLM frameworks and tools: - LangChain - LlamaIndex - Haystack - OpenAI\\n\\nObservability platforms - Phoenix - LangSmith - Langfuse\\n\\n📊 Comprehensive Analysis\\n\\nBeyond simple scores, Ragas provides detailed insights into your application\\'s strengths and weaknesses, enabling targeted improvements.\\n\\nGetting Started with Ragas\\n\\nInstalling Ragas is straightforward:\\n\\nbash uv init && uv add ragas\\n\\nHere\\'s a simple example of evaluating a response using Ragas:\\n\\n```python from ragas.metrics import Faithfulness from ragas.evaluation import EvaluationDataset from ragas.dataset_schema import SingleTurnSample from langchain_openai import ChatOpenAI from ragas.llms import LangchainLLMWrapper from langchain_openai import ChatOpenAI\\n\\nInitialize the LLM, you are going to new OPENAI API key\\n\\nevaluator_llm = LangchainLLMWrapper(ChatOpenAI(model=\"gpt-4o\"))\\n\\nYour evaluation data\\n\\ntest_data = { \"user_input\": \"What is the capital of France?\", \"retrieved_contexts\": [\"Paris is the capital and most populous city of France.\"], \"response\": \"The capital of France is Paris.\" }\\n\\nCreate a sample\\n\\nsample = SingleTurnSample(**test_data) # Unpack the dictionary into the constructor\\n\\nCreate metric\\n\\nfaithfulness = Faithfulness(llm=evaluator_llm)\\n\\nCalculate the score\\n\\nresult = await faithfulness.single_turn_ascore(sample) print(f\"Faithfulness score: {result}\") ```\\n\\n💡 Try it yourself: Explore the hands-on notebook for this workflow: 01_Introduction_to_Ragas\\n\\nWhat\\'s Coming in This Blog Series\\n\\nThis introduction is just the beginning. In the upcoming posts, we\\'ll dive deeper into all aspects of evaluating LLM applications with Ragas:\\n\\nPart 2: Basic Evaluation Workflow We\\'ll explore each metric in detail, explaining when and how to use them effectively.\\n\\nPart 3: Evaluating RAG Systems Learn specialized techniques for evaluating retrieval-augmented generation systems, including context precision, recall, and relevance.\\n\\nPart 4: Test Data Generation Discover how to create high-quality test datasets that thoroughly exercise your application\\'s capabilities.\\n\\nPart 5: Advanced Evaluation Techniques Go beyond basic metrics with custom evaluations, multi-aspect analysis, and domain-specific assessments.\\n\\nPart 6: Evaluating AI Agents Learn how to evaluate complex AI agents that engage in multi-turn interactions, use tools, and work toward specific goals.\\n\\nPart 7: Integrations and Observability Connect Ragas with your existing tools and platforms for streamlined evaluation workflows.\\n\\nPart 8: Building Feedback Loops Learn how to implement feedback loops that drive continuous improvement in your LLM applications. Transform evaluation insights into concrete improvements for your LLM applications.\\n\\nConclusion\\n\\nIn a world increasingly powered by LLMs, robust evaluation is the difference between reliable applications and unpredictable ones. Ragas provides the tools you need to confidently assess and improve your LLM applications.\\n\\nReady to Elevate Your LLM Applications?\\n\\nStart exploring Ragas today by visiting the official documentation. Share your thoughts, challenges, or success stories. If you\\'re facing specific evaluation hurdles, don\\'t hesitate to reach out—we\\'d love to help!')"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
-    "## Update Blog Data Process\n",
-    "\n",
-    "This process will:\n",
-    "1. Load existing blog posts\n",
-    "2. Process and update metadata\n",
-    "3. Create or update vector embeddings"
+    "docs[0]\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "3d56f688",
+   "execution_count": 11,
+   "id": "72dd14b5",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Process blog posts and create/update embeddings\n",
-    "result = utils.process_blog_posts(\n",
-    "    data_dir=DATA_DIR,\n",
-    "    create_embeddings=True,\n",
-    "    force_recreate_embeddings=FORCE_RECREATE_EMBEDDINGS\n",
-    ")\n",
-    "\n",
-    "# Access the documents and vector store\n",
-    "documents = result[\"documents\"]\n",
-    "stats = result[\"stats\"]\n",
-    "vector_store = result[\"vector_store\"]\n",
-    "\n",
-    "print(f\"\\nProcessed {len(documents)} blog posts\")\n",
-    "print(f\"Vector store created/updated at: {VECTOR_STORAGE_PATH}\")"
+    "vector_store = blog_utils = blog_utils.create_vector_store(docs,'./db/vector_store_4')"
    ]
   },
   {
@@ -168,13 +117,44 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "id": "8b552e6b",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Query: What is RAGAS?\n",
+      "Retrieved 3 documents:\n",
+      "1. Introduction To Ragas (https://thedataguy.pro/blog/introduction-to-ragas/)\n",
+      "2. Evaluating Rag Systems With Ragas (https://thedataguy.pro/blog/evaluating-rag-systems-with-ragas/)\n",
+      "3. Advanced Metrics And Customization With Ragas (https://thedataguy.pro/blog/advanced-metrics-and-customization-with-ragas/)\n",
+      "\n",
+      "Query: How to build research agents?\n",
+      "Retrieved 3 documents:\n",
+      "1. Building Research Agent (https://thedataguy.pro/blog/building-research-agent/)\n",
+      "2. Advanced Metrics And Customization With Ragas (https://thedataguy.pro/blog/advanced-metrics-and-customization-with-ragas/)\n",
+      "3. Evaluating Rag Systems With Ragas (https://thedataguy.pro/blog/evaluating-rag-systems-with-ragas/)\n",
+      "\n",
+      "Query: What is metric driven development?\n",
+      "Retrieved 3 documents:\n",
+      "1. Metric Driven Development (https://thedataguy.pro/blog/metric-driven-development/)\n",
+      "2. Advanced Metrics And Customization With Ragas (https://thedataguy.pro/blog/advanced-metrics-and-customization-with-ragas/)\n",
+      "3. Coming Back To Ai Roots (https://thedataguy.pro/blog/coming-back-to-ai-roots/)\n",
+      "\n",
+      "Query: Who is TheDataGuy?\n",
+      "Retrieved 3 documents:\n",
+      "1. Advanced Metrics And Customization With Ragas (https://thedataguy.pro/blog/advanced-metrics-and-customization-with-ragas/)\n",
+      "2. Langchain Experience Csharp Perspective (https://thedataguy.pro/blog/langchain-experience-csharp-perspective/)\n",
+      "3. Evaluating Rag Systems With Ragas (https://thedataguy.pro/blog/evaluating-rag-systems-with-ragas/)\n"
+     ]
+    }
+   ],
    "source": [
     "# Create a retriever from the vector store\n",
-    "retriever = vector_store.as_retriever(search_kwargs={\"k\": 2})\n",
+    "retriever = vector_store.as_retriever(search_kwargs={\"k\": 3})\n",
     "\n",
     "# Test queries\n",
     "test_queries = [\n",
@@ -194,49 +174,14 @@
     "        print(f\"{i+1}. {title} ({url})\")"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "ddbe9282",
-   "metadata": {},
-   "source": [
-    "## Schedule This Notebook\n",
-    "\n",
-    "To keep the blog data up-to-date, you can schedule this notebook to run periodically. \n",
-    "Here are some options:\n",
-    "\n",
-    "1. Use a cron job to run this notebook with papermill\n",
-    "2. Set up a GitHub Action to run this notebook on a schedule\n",
-    "3. Use Airflow or another workflow management system\n",
-    "\n",
-    "Example of running with papermill:\n",
-    "```bash\n",
-    "papermill update_blog_data.ipynb output_$(date +%Y%m%d).ipynb\n",
-    "```"
-   ]
-  },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "3634e064",
+   "execution_count": 13,
+   "id": "4cdd6899",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Save stats to a file for tracking changes over time\n",
-    "import json\n",
-    "from datetime import datetime\n",
-    "\n",
-    "stats_dir = Path(\"stats\")\n",
-    "stats_dir.mkdir(exist_ok=True)\n",
-    "\n",
-    "# Add timestamp to stats\n",
-    "stats[\"timestamp\"] = datetime.now().isoformat()\n",
-    "\n",
-    "# Save stats\n",
-    "stats_path = stats_dir / f\"blog_stats_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json\"\n",
-    "with open(stats_path, \"w\") as f:\n",
-    "    json.dump(stats, f, indent=2)\n",
-    "\n",
-    "print(f\"Saved stats to {stats_path}\")"
+    "vector_store.client.close()"
    ]
   }
  ],
@@ -247,7 +192,15 @@
    "name": "python3"
   },
   "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
    "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
    "version": "3.13.2"
   }
  },

update_blog_data.py

@@ -0,0 +1,101 @@
+"""
+Blog Data Update Script
+
+This script updates the blog data vector store when new posts are added.
+It can be scheduled to run periodically or manually executed.
+
+Usage:
+    python update_blog_data.py [--force-recreate] [--data-dir DATA_DIR]
+
+Options:
+    --force-recreate   Force recreation of the vector store even if it exists
+    --data-dir DIR     Directory containing the blog posts (default: data/)
+"""
+
+import os
+import sys
+import argparse
+from datetime import datetime
+import json
+from pathlib import Path
+
+# Import the blog utilities module
+import blog_utils
+
+def parse_args():
+    """Parse command-line arguments"""
+    parser = argparse.ArgumentParser(description="Update blog data vector store")
+    parser.add_argument("--force-recreate", action="store_true", 
+                        help="Force recreation of the vector store")
+    parser.add_argument("--data-dir", default=blog_utils.DATA_DIR,
+                        help=f"Directory containing blog posts (default: {blog_utils.DATA_DIR})")
+    return parser.parse_args()
+
+def save_stats(stats, output_dir="./stats"):
+    """Save stats to a JSON file for tracking changes over time"""
+    # Create directory if it doesn't exist
+    Path(output_dir).mkdir(exist_ok=True, parents=True)
+    
+    # Create filename with timestamp
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"{output_dir}/blog_stats_{timestamp}.json"
+    
+    # Save only the basic stats, not the full document list
+    basic_stats = {
+        "timestamp": timestamp,
+        "total_documents": stats["total_documents"],
+        "total_characters": stats["total_characters"],
+        "min_length": stats["min_length"],
+        "max_length": stats["max_length"],
+        "avg_length": stats["avg_length"],
+    }
+    
+    with open(filename, "w") as f:
+        json.dump(basic_stats, f, indent=2)
+    
+    print(f"Saved stats to {filename}")
+    return filename
+
+def main():
+    """Main function to update blog data"""
+    args = parse_args()
+    
+    print("=== Blog Data Update ===")
+    print(f"Data directory: {args.data_dir}")
+    print(f"Force recreate: {args.force_recreate}")
+    print("========================")
+    
+    # Process blog posts without creating embeddings
+    try:
+        # Load and process documents
+        documents = blog_utils.load_blog_posts(args.data_dir)
+        documents = blog_utils.update_document_metadata(documents)
+        
+        # Get stats
+        stats = blog_utils.get_document_stats(documents)
+        blog_utils.display_document_stats(stats)
+        
+        # Save stats for tracking
+        stats_file = save_stats(stats)
+        
+        # Create a reference file for the vector store
+        if args.force_recreate:
+            print("\nAttempting to save vector store reference file...")
+            blog_utils.create_vector_store(documents, force_recreate=args.force_recreate)
+        
+        print("\n=== Update Summary ===")
+        print(f"Processed {stats['total_documents']} documents")
+        print(f"Stats saved to: {stats_file}")
+        print("Note: Vector store creation is currently disabled due to pickling issues.")
+        print("      See VECTOR_STORE_ISSUES.md for more information and possible solutions.")
+        print("=====================")
+        
+        return 0
+    except Exception as e:
+        print(f"Error: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+
+if __name__ == "__main__":
+    sys.exit(main())

utils_data_loading.ipynb

@@ -1,454 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "b31c2849",
-   "metadata": {},
-   "source": [
-    "# Utility Functions for Blog Post Loading and Processing\n",
-    "\n",
-    "This notebook contains utility functions for loading blog posts from the data directory, processing their metadata, and creating vector embeddings for use in the RAG system."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "848b0a86",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import json\n",
-    "from pathlib import Path\n",
-    "from typing import List, Dict, Any, Optional\n",
-    "\n",
-    "from langchain_community.document_loaders import DirectoryLoader\n",
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
-    "from langchain.schema.document import Document\n",
-    "from langchain_huggingface import HuggingFaceEmbeddings\n",
-    "from langchain_community.vectorstores import Qdrant\n",
-    "\n",
-    "from IPython.display import Markdown, display\n",
-    "from dotenv import load_dotenv\n",
-    "\n",
-    "# Load environment variables from .env file\n",
-    "load_dotenv()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "39e32435",
-   "metadata": {},
-   "source": [
-    "## Configuration\n",
-    "\n",
-    "Load configuration from environment variables or use defaults."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "5a6a5d6d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Configuration with defaults\n",
-    "DATA_DIR = os.environ.get(\"DATA_DIR\", \"data/\")\n",
-    "VECTOR_STORAGE_PATH = os.environ.get(\"VECTOR_STORAGE_PATH\", \"./db/vectorstore_v3\")\n",
-    "EMBEDDING_MODEL = os.environ.get(\"EMBEDDING_MODEL\", \"Snowflake/snowflake-arctic-embed-l\")\n",
-    "QDRANT_COLLECTION = os.environ.get(\"QDRANT_COLLECTION\", \"thedataguy_documents\")\n",
-    "BLOG_BASE_URL = os.environ.get(\"BLOG_BASE_URL\", \"https://thedataguy.pro/blog/\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "01454147",
-   "metadata": {},
-   "source": [
-    "## Utility Functions\n",
-    "\n",
-    "These functions handle the loading, processing, and storing of blog posts."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "25792cd5",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def load_blog_posts(data_dir: str = DATA_DIR, \n",
-    "                   glob_pattern: str = \"*.md\", \n",
-    "                   recursive: bool = True, \n",
-    "                   show_progress: bool = True) -> List[Document]:\n",
-    "    \"\"\"\n",
-    "    Load blog posts from the specified directory.\n",
-    "    \n",
-    "    Args:\n",
-    "        data_dir: Directory containing the blog posts\n",
-    "        glob_pattern: Pattern to match files\n",
-    "        recursive: Whether to search subdirectories\n",
-    "        show_progress: Whether to show a progress bar\n",
-    "        \n",
-    "    Returns:\n",
-    "        List of Document objects containing the blog posts\n",
-    "    \"\"\"\n",
-    "    text_loader = DirectoryLoader(\n",
-    "        data_dir, \n",
-    "        glob=glob_pattern, \n",
-    "        show_progress=show_progress,\n",
-    "        recursive=recursive\n",
-    "    )\n",
-    "    \n",
-    "    documents = text_loader.load()\n",
-    "    print(f\"Loaded {len(documents)} documents from {data_dir}\")\n",
-    "    return documents"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e7ddba72",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def update_document_metadata(documents: List[Document], \n",
-    "                           data_dir_prefix: str = DATA_DIR,\n",
-    "                           blog_base_url: str = BLOG_BASE_URL,\n",
-    "                           remove_suffix: str = \"index.md\") -> List[Document]:\n",
-    "    \"\"\"\n",
-    "    Update the metadata of documents to include URL and other information.\n",
-    "    \n",
-    "    Args:\n",
-    "        documents: List of Document objects to update\n",
-    "        data_dir_prefix: Prefix to replace in source paths\n",
-    "        blog_base_url: Base URL for the blog posts\n",
-    "        remove_suffix: Suffix to remove from paths (like index.md)\n",
-    "        \n",
-    "    Returns:\n",
-    "        Updated list of Document objects\n",
-    "    \"\"\"\n",
-    "    for doc in documents:\n",
-    "        # Create URL from source path\n",
-    "        doc.metadata[\"url\"] = doc.metadata[\"source\"].replace(data_dir_prefix, blog_base_url)\n",
-    "        \n",
-    "        # Remove index.md or other suffix if present\n",
-    "        if remove_suffix and doc.metadata[\"url\"].endswith(remove_suffix):\n",
-    "            doc.metadata[\"url\"] = doc.metadata[\"url\"][:-len(remove_suffix)]\n",
-    "            \n",
-    "        # Extract post title from the directory structure\n",
-    "        path_parts = Path(doc.metadata[\"source\"]).parts\n",
-    "        if len(path_parts) > 1:\n",
-    "            # Use the directory name as post_slug\n",
-    "            doc.metadata[\"post_slug\"] = path_parts[-2]\n",
-    "            doc.metadata[\"post_title\"] = path_parts[-2].replace(\"-\", \" \").title()\n",
-    "            \n",
-    "        # Add document length as metadata\n",
-    "        doc.metadata[\"content_length\"] = len(doc.page_content)\n",
-    "    \n",
-    "    return documents"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e0dfe498",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def get_document_stats(documents: List[Document]) -> Dict[str, Any]:\n",
-    "    \"\"\"\n",
-    "    Get statistics about the documents.\n",
-    "    \n",
-    "    Args:\n",
-    "        documents: List of Document objects\n",
-    "        \n",
-    "    Returns:\n",
-    "        Dictionary with statistics\n",
-    "    \"\"\"\n",
-    "    stats = {\n",
-    "        \"total_documents\": len(documents),\n",
-    "        \"total_characters\": sum(len(doc.page_content) for doc in documents),\n",
-    "        \"min_length\": min(len(doc.page_content) for doc in documents),\n",
-    "        \"max_length\": max(len(doc.page_content) for doc in documents),\n",
-    "        \"avg_length\": sum(len(doc.page_content) for doc in documents) / len(documents) if documents else 0,\n",
-    "    }\n",
-    "    \n",
-    "    # Create a list of document info for analysis\n",
-    "    doc_info = []\n",
-    "    for doc in documents:\n",
-    "        doc_info.append({\n",
-    "            \"url\": doc.metadata.get(\"url\", \"\"),\n",
-    "            \"source\": doc.metadata.get(\"source\", \"\"),\n",
-    "            \"title\": doc.metadata.get(\"post_title\", \"\"),\n",
-    "            \"text_length\": doc.metadata.get(\"content_length\", 0),\n",
-    "        })\n",
-    "    \n",
-    "    stats[\"documents\"] = doc_info\n",
-    "    return stats"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0ae139c0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def display_document_stats(stats: Dict[str, Any]):\n",
-    "    \"\"\"\n",
-    "    Display document statistics in a readable format.\n",
-    "    \n",
-    "    Args:\n",
-    "        stats: Dictionary with statistics from get_document_stats\n",
-    "    \"\"\"\n",
-    "    print(f\"Total Documents: {stats['total_documents']}\")\n",
-    "    print(f\"Total Characters: {stats['total_characters']}\")\n",
-    "    print(f\"Min Length: {stats['min_length']} characters\")\n",
-    "    print(f\"Max Length: {stats['max_length']} characters\")\n",
-    "    print(f\"Average Length: {stats['avg_length']:.2f} characters\")\n",
-    "    \n",
-    "    # Display documents as a table\n",
-    "    import pandas as pd\n",
-    "    if stats[\"documents\"]:\n",
-    "        df = pd.DataFrame(stats[\"documents\"])\n",
-    "        display(df)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2dcf66b4",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def split_documents(documents: List[Document], \n",
-    "                   chunk_size: int = 1000, \n",
-    "                   chunk_overlap: int = 200) -> List[Document]:\n",
-    "    \"\"\"\n",
-    "    Split documents into chunks for better embedding and retrieval.\n",
-    "    \n",
-    "    Args:\n",
-    "        documents: List of Document objects to split\n",
-    "        chunk_size: Size of each chunk in characters\n",
-    "        chunk_overlap: Overlap between chunks in characters\n",
-    "        \n",
-    "    Returns:\n",
-    "        List of split Document objects\n",
-    "    \"\"\"\n",
-    "    text_splitter = RecursiveCharacterTextSplitter(\n",
-    "        chunk_size=chunk_size,\n",
-    "        chunk_overlap=chunk_overlap,\n",
-    "        length_function=len,\n",
-    "    )\n",
-    "    \n",
-    "    split_docs = text_splitter.split_documents(documents)\n",
-    "    print(f\"Split {len(documents)} documents into {len(split_docs)} chunks\")\n",
-    "    return split_docs"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "527ad848",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def create_vector_store(documents: List[Document], \n",
-    "                       storage_path: str = VECTOR_STORAGE_PATH,\n",
-    "                       collection_name: str = QDRANT_COLLECTION,\n",
-    "                       embedding_model: str = EMBEDDING_MODEL,\n",
-    "                       force_recreate: bool = False) -> Qdrant:\n",
-    "    \"\"\"\n",
-    "    Create a vector store from documents.\n",
-    "    \n",
-    "    Args:\n",
-    "        documents: List of Document objects to store\n",
-    "        storage_path: Path to the vector store\n",
-    "        collection_name: Name of the collection\n",
-    "        embedding_model: Name of the embedding model\n",
-    "        force_recreate: Whether to force recreation of the vector store\n",
-    "        \n",
-    "    Returns:\n",
-    "        Qdrant vector store\n",
-    "    \"\"\"\n",
-    "    # Initialize the embedding model\n",
-    "    embeddings = HuggingFaceEmbeddings(model_name=embedding_model)\n",
-    "    \n",
-    "    # Create the directory if it doesn't exist\n",
-    "    storage_dir = Path(storage_path).parent\n",
-    "    os.makedirs(storage_dir, exist_ok=True)\n",
-    "    \n",
-    "    # Check if vector store exists\n",
-    "    vector_store_exists = Path(storage_path).exists() and not force_recreate\n",
-    "    \n",
-    "    if vector_store_exists:\n",
-    "        print(f\"Loading existing vector store from {storage_path}\")\n",
-    "        try:\n",
-    "            vector_store = Qdrant(\n",
-    "                path=storage_path,\n",
-    "                embedding_function=embeddings,\n",
-    "                collection_name=collection_name\n",
-    "            )\n",
-    "            return vector_store\n",
-    "        except Exception as e:\n",
-    "            print(f\"Error loading existing vector store: {e}\")\n",
-    "            print(\"Creating new vector store...\")\n",
-    "            force_recreate = True\n",
-    "    \n",
-    "    # Create new vector store\n",
-    "    print(f\"Creating new vector store at {storage_path}\")\n",
-    "    vector_store = Qdrant.from_documents(\n",
-    "        documents=documents,\n",
-    "        embedding=embeddings,\n",
-    "        path=storage_path,\n",
-    "        collection_name=collection_name,\n",
-    "    )\n",
-    "    \n",
-    "    return vector_store"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "c78f99fc",
-   "metadata": {},
-   "source": [
-    "## Example Usage\n",
-    "\n",
-    "Here's how to use these utility functions for processing blog posts."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "132d32c6",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def process_blog_posts(data_dir: str = DATA_DIR,\n",
-    "                      create_embeddings: bool = True,\n",
-    "                      force_recreate_embeddings: bool = False):\n",
-    "    \"\"\"\n",
-    "    Complete pipeline to process blog posts and optionally create vector embeddings.\n",
-    "    \n",
-    "    Args:\n",
-    "        data_dir: Directory containing the blog posts\n",
-    "        create_embeddings: Whether to create vector embeddings\n",
-    "        force_recreate_embeddings: Whether to force recreation of embeddings\n",
-    "        \n",
-    "    Returns:\n",
-    "        Dictionary with data and vector store (if created)\n",
-    "    \"\"\"\n",
-    "    # Load documents\n",
-    "    documents = load_blog_posts(data_dir)\n",
-    "    \n",
-    "    # Update metadata\n",
-    "    documents = update_document_metadata(documents)\n",
-    "    \n",
-    "    # Get and display stats\n",
-    "    stats = get_document_stats(documents)\n",
-    "    display_document_stats(stats)\n",
-    "    \n",
-    "    result = {\n",
-    "        \"documents\": documents,\n",
-    "        \"stats\": stats,\n",
-    "        \"vector_store\": None\n",
-    "    }\n",
-    "    \n",
-    "    # Create vector store if requested\n",
-    "    if create_embeddings:\n",
-    "        vector_store = create_vector_store(\n",
-    "            documents, \n",
-    "            force_recreate=force_recreate_embeddings\n",
-    "        )\n",
-    "        result[\"vector_store\"] = vector_store\n",
-    "    \n",
-    "    return result"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "266d4fb3",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Example usage\n",
-    "if __name__ == \"__main__\":\n",
-    "    # Process blog posts without creating embeddings\n",
-    "    result = process_blog_posts(create_embeddings=False)\n",
-    "    \n",
-    "    # Example: Access the documents\n",
-    "    print(f\"\\nDocument example: {result['documents'][0].metadata}\")\n",
-    "    \n",
-    "    # Create embeddings if needed\n",
-    "    # result = process_blog_posts(create_embeddings=True)\n",
-    "    \n",
-    "    # Retriever example\n",
-    "    # retriever = result[\"vector_store\"].as_retriever()\n",
-    "    # query = \"What is RAGAS?\"\n",
-    "    # docs = retriever.invoke(query, k=2)\n",
-    "    # print(f\"\\nRetrieved {len(docs)} documents for query: {query}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "22132649",
-   "metadata": {},
-   "source": [
-    "## Function for Loading Existing Vector Store\n",
-    "\n",
-    "This function can be used to load an existing vector store without reprocessing all blog posts."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c24e0c02",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def load_vector_store(storage_path: str = VECTOR_STORAGE_PATH,\n",
-    "                     collection_name: str = QDRANT_COLLECTION,\n",
-    "                     embedding_model: str = EMBEDDING_MODEL) -> Optional[Qdrant]:\n",
-    "    \"\"\"\n",
-    "    Load an existing vector store.\n",
-    "    \n",
-    "    Args:\n",
-    "        storage_path: Path to the vector store\n",
-    "        collection_name: Name of the collection\n",
-    "        embedding_model: Name of the embedding model\n",
-    "        \n",
-    "    Returns:\n",
-    "        Qdrant vector store or None if it doesn't exist\n",
-    "    \"\"\"\n",
-    "    # Initialize the embedding model\n",
-    "    embeddings = HuggingFaceEmbeddings(model_name=embedding_model)\n",
-    "    \n",
-    "    # Check if vector store exists\n",
-    "    if not Path(storage_path).exists():\n",
-    "        print(f\"Vector store not found at {storage_path}\")\n",
-    "        return None\n",
-    "    \n",
-    "    try:\n",
-    "        vector_store = Qdrant(\n",
-    "            path=storage_path,\n",
-    "            embedding_function=embeddings,\n",
-    "            collection_name=collection_name\n",
-    "        )\n",
-    "        print(f\"Loaded vector store from {storage_path}\")\n",
-    "        return vector_store\n",
-    "    except Exception as e:\n",
-    "        print(f\"Error loading vector store: {e}\")\n",
-    "        return None"
-   ]
-  }
- ],
- "metadata": {
-  "language_info": {
-   "name": "python"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}