first commit in new repo

.gitignore

@@ -0,0 +1,10 @@
+.env
+__pycache__/
+*.pyc
+.DS_Store
+data/processed/
+*.csv
+# Ignore other PDFs and binary files except grid_code.pdf
+*.pdf
+!data/raw/grid_code.pdf
+*.docx

Dockerfile

@@ -0,0 +1,21 @@
+# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+FROM python:3.9
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+# Copy only the midterm project files
+COPY --chown=user requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Copy the midterm directory contents
+COPY --chown=user midterm /app
+
+# Set environment variable for Python path
+ENV PYTHONPATH=/app/src
+
+# Run streamlit on port 7860 for Hugging Face Spaces
+CMD ["streamlit", "run", "src/ui/app.py", "--server.port", "7860", "--server.address", "0.0.0.0"]

app.py

@@ -0,0 +1,11 @@
+import sys
+from pathlib import Path
+
+# Add src directory to Python path
+src_path = Path(__file__).parent / "src"
+sys.path.append(str(src_path))
+
+from ui.app import main
+
+if __name__ == "__main__":
+    main() 

app/.gitattributes

@@ -0,0 +1,3 @@
+data/raw/grid_code.pdf filter=lfs diff=lfs merge=lfs -text 
+*.docx filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text

app/.hf/space

@@ -0,0 +1,6 @@
+title: "GridGuide: Field Assistant"
+emoji: 🌩️
+colorFrom: blue
+colorTo: indigo
+sdk: docker
+pinned: false 

app/data/README.md

@@ -0,0 +1,28 @@
+# Data Directory
+
+This directory contains the Grid Code documentation and processed data.
+
+## Structure
+
+- `raw/` - Contains the original Grid Code PDF
+- `processed/` - Contains processed chunks and embeddings
+- `test/` - Contains test data and evaluation sets
+
+## Grid Code PDF
+
+Place the Grid Code PDF file in the `raw/` directory with filename `grid_code.pdf`.
+
+## Processing
+
+The data processing pipeline:
+1. Loads PDF from raw/
+2. Splits into chunks
+3. Generates embeddings
+4. Stores processed data
+
+## Test Data
+
+The test directory contains:
+- Sample questions and answers
+- Evaluation datasets
+- Test PDF segments 

app/data/processed/qdrant/.lock

@@ -0,0 +1 @@
+tmp lock file

app/data/processed/qdrant/meta.json

@@ -0,0 +1 @@
+{"collections": {"grid_code": {"vectors": {"size": 1536, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null, "strict_mode_config": null}}, "aliases": {}}

app/requirements/dev.txt

@@ -0,0 +1,13 @@
+langchain-community==0.3.14
+langchain-openai==0.2.14
+langchain-qdrant>=0.2.0
+openai>=1.6.1
+qdrant-client>=1.6.4
+ragas==0.2.10
+streamlit==1.29.0
+python-dotenv==1.0.0
+pypdf==3.17.1
+rich>=13.7.0
+rapidfuzz>=3.6.1
+tenacity>=8.2.3 
+sentence-transformers==3.4.1

app/requirements/prod.txt

@@ -0,0 +1 @@
+-r dev.txt 

app/src/data/__init__.py

@@ -0,0 +1 @@
+# Data handling utilities 

app/src/data/pdf_loader.py

@@ -0,0 +1,22 @@
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+class GridCodePDFLoader:
+    def __init__(self, pdf_path):
+        self.pdf_path = pdf_path
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=500,
+            chunk_overlap=50,
+            separators=["\n\n", "\n", ".", " ", ""]
+        )
+    
+    def load_and_split(self):
+        """Load PDF and split into chunks"""
+        loader = PyPDFLoader(self.pdf_path)
+        pages = loader.load()
+        return self.text_splitter.split_documents(pages)
+    
+    def extract_metadata(self):
+        """Extract metadata from PDF like sections, tables etc."""
+        # TODO: Implement metadata extraction
+        pass 

app/src/data/processed/qdrant/.lock

@@ -0,0 +1 @@
+tmp lock file

app/src/data/processed/qdrant/meta.json

@@ -0,0 +1 @@
+{"collections": {}, "aliases": {}}

app/src/embedding/fine_tune.py

@@ -0,0 +1,50 @@
+# fine_tune_lama.py
+
+import pandas as pd
+import logging
+from sentence_transformers import InputExample, SentenceTransformer, losses
+from torch.utils.data import DataLoader
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Load the embedding model
+model_id = "Snowflake/snowflake-arctic-embed-l"  
+logging.info(f"Loading model: {model_id}")
+model = SentenceTransformer(model_id)
+
+def load_synthetic_dataset():
+    logging.info("Loading synthetic dataset...")
+    df = pd.read_csv("../data/processed/synthetic_test_dataset.csv")
+    # Convert to the format expected by the model
+    examples = []
+    for _, row in df.iterrows():
+        examples.append(
+            InputExample(texts=[row["user_input"], row["reference"]], label=1)
+        )  # Assuming label 1 for positive pairs
+    logging.info(f"Loaded {len(examples)} examples.")
+    return examples
+
+train_examples = load_synthetic_dataset()
+train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=16)
+
+# Define the loss function
+inner_train_loss = losses.MultipleNegativesRankingLoss(model)
+train_loss = losses.MatryoshkaLoss(
+    model, inner_train_loss, matryoshka_dims=[768, 512, 256, 128, 64]
+)
+
+EPOCHS = 1
+warmup_steps = int(len(train_dataloader) * EPOCHS * 0.1)
+
+# Fine-tune the model
+logging.info("Starting model training...")
+model.fit(
+    train_objectives=[(train_dataloader, train_loss)],
+    epochs=EPOCHS,
+    warmup_steps=warmup_steps,
+    output_path="data/processed/finetuned_arctic_ft",
+    show_progress_bar=True,
+)
+
+logging.info("Model training completed.")

app/src/embedding/model.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_openai import OpenAIEmbeddings
+
+
+class EmbeddingModel:
+    def __init__(self, model_type="openai"):
+        self.model_type = model_type
+        self.model = self._initialize_model()
+
+    def _initialize_model(self):
+        if self.model_type == "openai":
+            return OpenAIEmbeddings(model="text-embedding-3-small")
+        elif self.model_type == "finetuned":
+            model_path = (
+                Path(__file__).parent.parent.parent
+                / "data"
+                / "processed"
+                / "finetuned_arctic_ft_repo"
+            )
+            return HuggingFaceEmbeddings(
+                model_name=str(model_path),
+                model_kwargs={"device": "cpu"},
+                encode_kwargs={"normalize_embeddings": True},
+            )
+        else:
+            raise ValueError(f"Unsupported model type: {self.model_type}")
+
+    def embed_documents(self, texts):
+        """Embed a list of texts"""
+        return self.model.embed_documents(texts)
+
+    def embed_query(self, text):
+        """Embed a single text"""
+        return self.model.embed_query(text)

app/src/embedding/save_to_hf.py

@@ -0,0 +1,51 @@
+### `save_to_hf.py`
+
+import logging
+import os
+
+from huggingface_hub import HfApi, Repository
+
+# Set up logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+
+
+def push_model_to_huggingface(model_dir, model_name, hf_username):
+    """Push the model to Hugging Face Hub using the Repository class."""
+    try:
+        # Create a new directory for the repository
+        repo_dir = f"./{model_name}_repo"  # Specify a new directory
+        os.makedirs(repo_dir, exist_ok=True)
+
+        # Initialize the repository
+        repo_id = f"{hf_username}/{model_name}"
+        repo = Repository(local_dir=repo_dir, clone_from=repo_id)
+
+        # Copy model files to the new repository directory
+        for filename in os.listdir(model_dir):
+            full_file_name = os.path.join(model_dir, filename)
+            if os.path.isfile(full_file_name):
+                os.rename(full_file_name, os.path.join(repo_dir, filename))
+
+        # Add model files to the repository
+        repo.git_add()
+        repo.git_commit("Add custom segmentation model")
+        repo.git_push()
+
+        logging.info(f"Model pushed to Hugging Face Hub: {repo_id}")
+
+    except Exception as e:
+        logging.error(f"Failed to push model to Hugging Face Hub: {str(e)}")
+
+
+if __name__ == "__main__":
+    # Define parameters
+    model_directory = (
+        "src/data/processed/finetuned_arctic_ft"  # Directory where the model is saved
+    )
+    model_name = "finetuned_arctic_ft"  # Name for the model on Hugging Face
+    hf_username = "vanessaprzybylo"  # Replace with your Hugging Face username
+
+    # Push the model to Hugging Face
+    push_model_to_huggingface(model_directory, model_name, hf_username)

app/src/evaluation/evaluate_rag.py

@@ -0,0 +1,208 @@
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+
+import pandas as pd
+from dotenv import load_dotenv
+from tenacity import retry, stop_after_attempt, wait_exponential
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Add src directory to Python path
+src_path = Path(__file__).parent.parent
+sys.path.append(str(src_path))
+
+# Load environment variables
+root_dir = Path(__file__).parent.parent.parent
+env_path = root_dir / ".env"
+load_dotenv(env_path)
+
+from embedding.model import EmbeddingModel
+from langchain.chat_models import init_chat_model
+from langchain_core.rate_limiters import InMemoryRateLimiter
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from rag.chain import RAGChain
+from rag.document_loader import GridCodeLoader
+from rag.vectorstore import VectorStore
+from ragas import EvaluationDataset, RunConfig, evaluate
+from ragas.embeddings import LangchainEmbeddingsWrapper
+from ragas.llms import LangchainLLMWrapper
+from ragas.metrics import AnswerRelevancy, ContextPrecision, ContextRecall, Faithfulness
+from ragas.testset import TestsetGenerator
+
+
+def setup_rag(embedding_model_type):
+    """Initialize RAG system for evaluation with specified embedding model."""
+    logger.info("Setting up RAG system...")
+
+    # Load documents
+    data_path = root_dir / "data" / "raw" / "grid_code.pdf"
+    if not data_path.exists():
+        raise FileNotFoundError(f"PDF not found: {data_path}")
+
+    loader = GridCodeLoader(str(data_path), pages=17)
+    documents = loader.load_and_split()
+    logger.info(f"Loaded {len(documents)} document chunks")
+
+    # Initialize embedding model and vectorstore
+    embedding_model = EmbeddingModel(model_type=embedding_model_type)
+    vectorstore = VectorStore(embedding_model)
+    vectorstore = vectorstore.create_vectorstore(documents)
+
+    return RAGChain(vectorstore), documents
+
+
+def generate_test_dataset(documents, n_questions=30):
+    """Generate synthetic test dataset using RAGAS or load it if it already exists."""
+    dataset_path = "../data/processed/synthetic_test_dataset.csv"
+
+    # Check if the dataset already exists
+    if os.path.exists(dataset_path):
+        logger.info(f"Loading existing synthetic test dataset from {dataset_path}...")
+        return pd.read_csv(dataset_path)
+
+    logger.info("Generating synthetic test dataset...")
+
+    # Initialize the rate limiter
+    rate_limiter = InMemoryRateLimiter(
+        requests_per_second=1,  # Make a request once every 1 second
+        check_every_n_seconds=0.1,  # Check every 100 ms to see if allowed to make a request
+        max_bucket_size=10,  # Controls the maximum burst size
+    )
+
+    # Initialize the chat model with the rate limiter
+    model = init_chat_model("gpt-4o", temperature=0, rate_limiter=rate_limiter)
+
+    # Initialize generator models
+    generator_llm = LangchainLLMWrapper(model)
+    generator_embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())
+
+    # Create test set generator
+    generator = TestsetGenerator(
+        llm=generator_llm, embedding_model=generator_embeddings
+    )
+
+    # Generate synthetic test dataset
+    dataset = generator.generate_with_langchain_docs(
+        documents, testset_size=n_questions
+    )
+
+    df = dataset.to_pandas()
+    df.to_csv(dataset_path, index=False)  # Save as CSV
+    logger.info(
+        f"Generated synthetic dataset with {len(df)} test cases and saved to '{dataset_path}'."
+    )
+    return df
+
+
+@retry(wait=wait_exponential(multiplier=1, min=4, max=60), stop=stop_after_attempt(5))
+def get_rag_response(rag_chain, question):
+    """Get RAG response with retry logic"""
+    return rag_chain.invoke(question)
+
+
+def evaluate_rag_system(rag_chain, test_dataset):
+    """Evaluate RAG system using RAGAS metrics"""
+    logger.info("Starting RAGAS evaluation...")
+
+    # Get RAG responses for each question
+    eval_data = []
+
+    # Iterate through DataFrame rows
+    for _, row in test_dataset.iterrows():
+        # Add delay between requests
+        time.sleep(3)  # Wait 3 seconds between requests
+        response = get_rag_response(rag_chain, row["user_input"])
+        eval_data.append(
+            {
+                "user_input": row["user_input"],
+                "response": response["answer"],
+                "retrieved_contexts": [doc.page_content for doc in response["context"]],
+                "ground_truth": row["reference"],  # Keep for faithfulness
+                "reference": row["reference"],  # Keep for context_recall
+            }
+        )
+        logger.info(f"Processed question: {row['user_input'][:50]}...")
+
+    # Convert to pandas then to EvaluationDataset
+    eval_df = pd.DataFrame(eval_data)
+    logger.info("Sample evaluation data:")
+    logger.info(eval_df.iloc[0].to_dict())
+    eval_dataset = EvaluationDataset.from_pandas(eval_df)
+
+    # Initialize RAGAS evaluator
+    evaluator_llm = LangchainLLMWrapper(ChatOpenAI(model="gpt-4o"))
+
+    custom_run_config = RunConfig(timeout=360, max_workers=32)
+
+    # Run evaluation
+    results = evaluate(
+        eval_dataset,
+        metrics=[
+            Faithfulness(),  # Measures how accurately the generated response reflects the ground truth.
+            AnswerRelevancy(),  # Assesses the relevance of the answer to the user's question.
+            ContextRecall(),  # Evaluates the ability of the model to retrieve relevant context from the documents.
+            ContextPrecision(),  # Measures the precision of the retrieved contexts in relation to the user's question.
+        ],
+        llm=evaluator_llm,
+        run_config=custom_run_config,
+    )
+
+    return results
+
+
+def run_evaluation_with_model(rag_chain, test_dataset, embedding_model_type):
+    """Run evaluation with the specified embedding model type."""
+    logger.info(f"Running evaluation with {embedding_model_type} embeddings...")
+
+    # Run evaluation
+    results = evaluate_rag_system(rag_chain, test_dataset)
+
+    logger.info(f"Evaluation Results for {embedding_model_type}:")
+    logger.info(results)
+
+    # Save results to CSV
+    results_path = Path("../data/processed/")
+    results_path.mkdir(parents=True, exist_ok=True)
+
+    # Convert results to DataFrame
+    results_df = pd.DataFrame([results])
+    results_df.to_csv(
+        results_path / f"evaluation_results_{embedding_model_type}.csv", index=False
+    )
+    logger.info(
+        f"Saved evaluation results to evaluation_results_{embedding_model_type}.csv"
+    )
+
+
+def main():
+    """Main evaluation script"""
+    logger.info("Starting RAG evaluation")
+
+    try:
+        # Setup RAG system with the fine-tuned embedding model
+        rag_chain_finetuned, documents = setup_rag("finetuned")
+
+        # Generate synthetic test dataset
+        test_dataset = generate_test_dataset(documents)
+
+        # Run evaluations with both embedding models
+        run_evaluation_with_model(rag_chain_finetuned, test_dataset, "finetuned")
+
+        # # Setup RAG system with the OpenAI embedding model
+        # rag_chain_openai, _ = setup_rag("openai")
+
+        # # Run evaluation with OpenAI embeddings
+        # run_evaluation_with_model(rag_chain_openai, test_dataset, "openai")
+
+    except Exception as e:
+        logger.error(f"Evaluation failed: {str(e)}")
+        raise
+
+
+if __name__ == "__main__":
+    main()

app/src/rag/chain.py

@@ -0,0 +1,30 @@
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_openai import ChatOpenAI
+from langchain.chains import create_retrieval_chain
+from langchain.chains.combine_documents import create_stuff_documents_chain
+
+class RAGChain:
+    def __init__(self, vectorstore):
+        self.vectorstore = vectorstore
+        self.llm = ChatOpenAI(model="gpt-4o")
+        self.chain = self._create_chain()
+    
+    def _create_chain(self):
+        prompt = ChatPromptTemplate.from_template("""
+        You are a helpful assistant for field workers in the electricity transmission sector.
+        Answer questions about the Grid Code using the following context.
+        If you're unsure or the context doesn't contain the answer, say so.
+        
+        Context: {context}
+        Question: {input}
+        """)
+        
+        document_chain = create_stuff_documents_chain(self.llm, prompt)
+        retrieval_chain = create_retrieval_chain(
+            self.vectorstore.as_retriever(), 
+            document_chain
+        )
+        return retrieval_chain
+    
+    def invoke(self, question):
+        return self.chain.invoke({"input": question}) 

app/src/rag/document_loader.py

@@ -0,0 +1,44 @@
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import pypdf
+import logging
+
+logger = logging.getLogger(__name__)
+
+class GridCodeLoader:
+    def __init__(self, file_path, pages=None):
+        self.file_path = file_path
+        self.pages = pages
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=2000,
+            chunk_overlap=50,
+            separators=["\n\n", "\n", ".", " ", ""]
+        )
+    
+    def load_and_split(self):
+        logger.info(f"Loading PDF from {self.file_path}")
+        # Open PDF directly first to get total pages
+        reader = pypdf.PdfReader(self.file_path)
+        total_pages = len(reader.pages)
+        
+        if isinstance(self.pages, int):
+            # Load first n pages
+            pages_to_load = list(range(min(self.pages, total_pages)))
+            logger.info(f"Loaded first {len(pages_to_load)} pages from PDF")
+        elif isinstance(self.pages, (list, tuple)):
+            # Load specific pages
+            pages_to_load = [p for p in self.pages if p < total_pages]
+            logger.info(f"Loaded pages {self.pages} from PDF")
+        else:
+            pages_to_load = list(range(total_pages))
+            logger.info(f"Loaded all {len(pages_to_load)} pages from PDF")
+        
+        # Now use PyPDFLoader with the selected pages
+        loader = PyPDFLoader(self.file_path)
+        documents = loader.load()
+        documents = [doc for i, doc in enumerate(documents) if i in pages_to_load]
+        
+        logger.info("Splitting documents into chunks...")
+        chunks = self.text_splitter.split_documents(documents)
+        logger.info(f"Created {len(chunks)} chunks")
+        return chunks 

app/src/rag/vectorstore.py

@@ -0,0 +1,27 @@
+import logging
+import os
+
+from langchain_community.vectorstores import Qdrant
+
+logger = logging.getLogger(__name__)
+
+
+class VectorStore:
+    def __init__(self, embedding_model):
+        self.embedding_model = embedding_model
+        self.collection_name = "grid_code"
+
+    def create_vectorstore(self, documents):
+        """Create vector store."""
+        logger.info("Creating vector store...")
+        vectorstore = Qdrant.from_documents(
+            documents=documents,
+            embedding=self.embedding_model.model,
+            location=":memory:",  # Use in-memory storage
+            collection_name=self.collection_name,
+        )
+        logger.info(f"Created vector store with {len(documents)} chunks")
+        return vectorstore
+
+    def similarity_search(self, query, k=4):
+        raise NotImplementedError("Use the Qdrant vectorstore instance directly")

app/src/ui/app.py

@@ -0,0 +1,290 @@
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import Annotated, TypedDict
+
+import requests
+import streamlit as st
+from langchain.agents import AgentExecutor, create_tool_calling_agent
+from langchain_core.prompts import (
+    ChatPromptTemplate,
+    HumanMessagePromptTemplate,
+    MessagesPlaceholder,
+    SystemMessagePromptTemplate,
+)
+from langchain_core.tools import Tool
+from langchain_openai import ChatOpenAI
+from langgraph.graph.message import add_messages
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Add src directory to Python path
+src_path = Path(__file__).parent.parent
+sys.path.append(str(src_path))
+
+
+# Get secrets from Hugging Face Space
+def get_secrets():
+    """Get secrets from Hugging Face Space or local environment."""
+    if os.environ.get("SPACE_ID"):
+        # We're in a Hugging Face Space
+        return {
+            "OPENAI_API_KEY": st.secrets["OPENAI_API_KEY"],
+            "TAVILY_API_KEY": st.secrets.get("TAVILY_API_KEY"),
+            "LANGCHAIN_API_KEY": st.secrets.get("LANGCHAIN_API_KEY"),
+            "LANGCHAIN_PROJECT": st.secrets.get("LANGCHAIN_PROJECT", "GridGuide"),
+            "LANGCHAIN_TRACING_V2": st.secrets.get("LANGCHAIN_TRACING_V2", "true"),
+        }
+    else:
+        # We're running locally, use environment variables
+        return {
+            "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"),
+            "TAVILY_API_KEY": os.environ.get("TAVILY_API_KEY"),
+            "LANGCHAIN_API_KEY": os.environ.get("LANGCHAIN_API_KEY"),
+            "LANGCHAIN_PROJECT": os.environ.get("LANGCHAIN_PROJECT", "GridGuide"),
+            "LANGCHAIN_TRACING_V2": os.environ.get("LANGCHAIN_TRACING_V2", "true"),
+        }
+
+
+# Set up environment variables from secrets
+secrets = get_secrets()
+for key, value in secrets.items():
+    if value:
+        os.environ[key] = value
+
+# Verify API keys
+if not os.environ.get("OPENAI_API_KEY"):
+    st.error(
+        "OpenAI API key not found. Please set it in the Hugging Face Space secrets."
+    )
+    st.stop()
+
+from embedding.model import EmbeddingModel
+from rag.chain import RAGChain
+from rag.document_loader import GridCodeLoader
+from rag.vectorstore import VectorStore
+
+
+class WeatherTool:
+    def __init__(self):
+        self.base_url = "https://api.weather.gov"
+        self.headers = {
+            "User-Agent": "(Grid Code Assistant, contact@example.com)",
+            "Accept": "application/json",
+        }
+
+    def get_coordinates_from_zip(self, zipcode):
+        response = requests.get(f"https://api.zippopotam.us/us/{zipcode}")
+        if response.status_code == 200:
+            data = response.json()
+            return {
+                "lat": data["places"][0]["latitude"],
+                "lon": data["places"][0]["longitude"],
+                "place": data["places"][0]["place name"],
+                "state": data["places"][0]["state"],
+            }
+        return None
+
+    def run(self, zipcode):
+        coords = self.get_coordinates_from_zip(zipcode)
+        if not coords:
+            return {"error": "Invalid ZIP code or unable to get coordinates."}
+
+        point_url = f"{self.base_url}/points/{coords['lat']},{coords['lon']}"
+        response = requests.get(point_url, headers=self.headers)
+
+        if response.status_code != 200:
+            return {"error": "Unable to fetch weather data."}
+
+        grid_data = response.json()
+        forecast_url = grid_data["properties"]["forecast"]
+
+        response = requests.get(forecast_url, headers=self.headers)
+        if response.status_code == 200:
+            forecast_data = response.json()["properties"]["periods"]
+            weather_data = {
+                "type": "weather",
+                "location": f"{coords['place']}, {coords['state']}",
+                "current": forecast_data[0],
+                "forecast": forecast_data[1:4],
+            }
+            # Save to session state
+            st.session_state.weather_data = weather_data
+            return weather_data
+        return {"error": "Unable to fetch forecast data."}
+
+
+def initialize_rag():
+    """Initialize RAG system."""
+    if "rag_chain" in st.session_state:
+        logger.info("Using cached RAG chain from session state")
+        return st.session_state.rag_chain
+
+    # Use relative path from src directory
+    data_path = Path(__file__).parent.parent.parent / "data" / "raw" / "grid_code.pdf"
+    if not data_path.exists():
+        raise FileNotFoundError(f"PDF not found: {data_path}")
+
+    with st.spinner("Loading Grid Code documents..."):
+        loader = GridCodeLoader(str(data_path), pages=17)
+        documents = loader.load_and_split()
+        logger.info(f"Loaded {len(documents)} document chunks")
+
+    with st.spinner("Creating vector store..."):
+        embedding_model = EmbeddingModel()
+        vectorstore = VectorStore(embedding_model)
+        vectorstore = vectorstore.create_vectorstore(documents)
+        logger.info("Vector store created successfully")
+
+    # Cache the RAG chain in session state
+    rag_chain = RAGChain(vectorstore)
+    st.session_state.rag_chain = rag_chain
+    return rag_chain
+
+
+class RAGTool:
+    def __init__(self, rag_chain):
+        self.rag_chain = rag_chain
+
+    def run(self, question: str) -> str:
+        """Answer questions using the Grid Code."""
+        response = self.rag_chain.invoke(question)
+        return response["answer"]
+
+
+class AgentState(TypedDict):
+    """State definition for the agent."""
+
+    messages: Annotated[list, add_messages]
+
+
+def create_agent_workflow(rag_chain, weather_tool):
+    """Create an agent that can use both RAG and weather tools."""
+
+    # Define the tools
+    tools = [
+        Tool(
+            name="grid_code_query",
+            description="Answer questions about the Grid Code and electrical regulations",
+            func=lambda q: rag_chain.invoke(q)["answer"],
+        ),
+        Tool(
+            name="get_weather",
+            description="Get weather forecast for a ZIP code. Input should be a 5-digit ZIP code.",
+            func=lambda z: weather_tool.run(z),
+        ),
+    ]
+
+    # Initialize the LLM
+    llm = ChatOpenAI(model="gpt-4o", temperature=0)
+
+    # Create the custom prompt
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            SystemMessagePromptTemplate.from_template(
+                """You are a helpful assistant that specializes in two areas:
+            1. Answering questions about electrical Grid Code regulations
+            2. Providing weather information for specific locations
+
+            For weather queries:
+            - Extract the ZIP code from the question
+            - Use the get_weather tool to fetch the forecast
+
+            For Grid Code questions:
+            - Use the grid_code_query tool to find relevant information
+            - If the information isn't in the Grid Code, clearly state that
+            - Provide specific references when possible
+            """
+            ),
+            MessagesPlaceholder(variable_name="chat_history", optional=True),
+            HumanMessagePromptTemplate.from_template("{input}"),
+            MessagesPlaceholder(variable_name="agent_scratchpad"),
+        ]
+    )
+
+    # Create the agent
+    agent = create_tool_calling_agent(llm, tools, prompt)
+
+    return AgentExecutor(
+        agent=agent,
+        tools=tools,
+        verbose=True,
+        handle_parsing_errors=True,
+    )
+
+
+def display_weather(weather_data):
+    """Display weather information in a nice format"""
+    if "error" in weather_data:
+        st.error(weather_data["error"])
+        return
+
+    if weather_data.get("type") == "weather":
+        # Location header
+        st.header(f"Weather for {weather_data['location']}")
+
+        # Current conditions
+        current = weather_data["current"]
+        st.subheader("Current Conditions")
+
+        # Use columns for current weather layout
+        col1, col2 = st.columns(2)
+
+        with col1:
+            # Temperature display with metric
+            st.metric(
+                "Temperature", f"{current['temperature']}°{current['temperatureUnit']}"
+            )
+            # Wind information
+            st.info(f"💨 Wind: {current['windSpeed']} {current['windDirection']}")
+
+        with col2:
+            # Current forecast
+            st.markdown(f"**🌤️ Conditions:** {current['shortForecast']}")
+            st.markdown(f"**📝 Details:** {current['detailedForecast']}")
+
+        # Extended forecast
+        st.subheader("Extended Forecast")
+        for period in weather_data["forecast"]:
+            with st.expander(f"📅 {period['name']}"):
+                st.markdown(
+                    f"**🌡️ Temperature:** {period['temperature']}°{period['temperatureUnit']}"
+                )
+                st.markdown(
+                    f"**💨 Wind:** {period['windSpeed']} {period['windDirection']}"
+                )
+                st.markdown(f"**🌤️ Forecast:** {period['shortForecast']}")
+                st.markdown(f"**📝 Details:** {period['detailedForecast']}")
+
+
+def main():
+    st.title("GridGuide: Field Assistant")
+
+    # Initialize if not in session state
+    if "app" not in st.session_state:
+        rag_chain = initialize_rag()
+        weather_tool = WeatherTool()
+        st.session_state.app = create_agent_workflow(rag_chain, weather_tool)
+
+    # Create the input box
+    user_input = st.text_input("Ask about weather or the Grid Code:")
+
+    if user_input:
+        with st.spinner("Processing your request..."):
+            # Invoke the agent executor
+            result = st.session_state.app.invoke({"input": user_input})
+
+            # Check if we have weather data in session state
+            if "weather_data" in st.session_state:
+                display_weather(st.session_state.weather_data)
+                # Clear the weather data after displaying
+                del st.session_state.weather_data
+            else:
+                st.write(result["output"])
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,14 @@
+streamlit>=1.32.0
+langchain==0.3.19
+langchain-core==0.3.37
+langchain-openai==0.3.6
+langchain-huggingface==0.1.2
+langchain-community>=0.3.14
+python-dotenv>=1.0.0
+requests>=2.31.0
+langgraph==0.2.74
+qdrant-client>=1.7.3
+pypdf>=4.0.1
+openai==1.64.0
+typing-extensions>=4.9.0
+pydantic>=2.6.3

setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="grid-code-assistant",
+    version="0.1",
+    packages=find_packages(),
+)