Completed transactions answering query

app/api/routers/transaction.py

@@ -6,6 +6,10 @@ from app.model.transaction import Transaction as TransactionModel
 from app.schema.index import TransactionResponse
 from app.engine.postgresdb import get_db_session
 
+from app.service.query_rag import answer_query, fetch_transaction_documents
+
+import pandas as pd
+
 transaction_router = r = APIRouter(prefix="/api/v1/transactions", tags=["transactions"])
 
 
@@ -59,11 +63,14 @@ async def answer_transactions_query(user_id: int, query: str, db: AsyncSession =
     """
     Retrieve all transactions.
     """
-    result = await TransactionModel.get_by_user(db, user_id)
-    all_rows = result.all()
-    print(f"answer_transactions_query: \n type: {type(all_rows)}\nall_rows: {all_rows}")
-    # TODO: Pass all rows to answer_query method in transaction_query_rag and get result that will be the
-    # string param in TransactionRagResponse
-    if len(all_rows) == 0:
-        raise HTTPException(status_code=500, detail="No transactions found for this user")
-    return all_rows
+    try:
+        result = await TransactionModel.get_by_user(db, user_id)
+        all_rows = result.all()
+        if len(all_rows) == 0:
+            raise HTTPException(status_code=500, detail="No transactions found for this user")
+        document_splits = await fetch_transaction_documents(all_rows)
+        answer = await answer_query(document_splits, query, user_id)
+        return answer
+    
+    except Exception as e:
+         raise HTTPException(status_code=500, detail=f"answer_transactions_query error: {str(e)}")

app/api/routers/user.py

@@ -28,14 +28,17 @@ logger = logging.getLogger(__name__)
 async def create_user(user: UserCreate, db: AsyncSession = Depends(get_db_session)):
     try:
         db_user = await UserModel.get(db, email=user.email)
+        print(f"db_user: {db_user}")
         if db_user and not db_user.is_deleted:
             raise HTTPException(status_code=409, detail="User already exists")
 
         await UserModel.create(db, **user.model_dump())
         user = await UserModel.get(db, email=user.email)
+        print(f"user: {user}")
 
         fake_transactions = get_fake_transactions(user.id)
         await TransactionModel.bulk_create(db, fake_transactions)
+        print(f"fake_transactions: {fake_transactions}")
 
         return user
     

app/model/transaction.py

@@ -23,6 +23,15 @@ class Transaction(Base, BaseModel):
 
     def __str__(self) -> str:
         return f"{self.transaction_date}, {self.category}, {self.name_description}, {self.amount}, {self.type}"
+    
+    def to_dict(self):
+        return {
+            'transaction_date': self.transaction_date,
+            'category': self.category,
+            'name_description': self.name_description,
+            'amount': self.amount,
+            'type': self.type
+        }
 
     @classmethod
     async def create(cls: "type[Transaction]", db: AsyncSession, **kwargs) -> "Transaction":

app/service/query_rag.py

@@ -0,0 +1,142 @@
+from pinecone import Pinecone, ServerlessSpec
+from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings, ChatOpenAI
+from langchain_pinecone import PineconeVectorStore
+from langchain.vectorstores import Chroma
+from langchain.chains import RetrievalQA
+from langchain.prompts import PromptTemplate
+from langchain_core.documents import Document
+
+from fastapi import HTTPException
+
+from typing import List
+from app.model.transaction import Transaction
+
+import os
+
+async def fetch_transaction_documents(transactions: List[Transaction]) -> List[Document]:
+    try:
+        document = ''.join(str(row)+'\n' for row in transactions)
+
+        page_splitter = CharacterTextSplitter(chunk_size=100, chunk_overlap=0)
+        pages = page_splitter.split_text(document)
+
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=10)
+        document_splits = text_splitter.create_documents(pages)
+
+        return document_splits
+    except Exception as e:
+        raise HTTPException(status_code = 500, detail=f"fetch_transaction_documents error: {str(e)}")
+    
+
+async def answer_query(document_splits: List[Document], query: str, user_id: int) -> str:
+    """Creates an embedding of the transactions table and then returns the answer for the given query.
+    Args:
+        df (pd.DataFrame): DataFrame containing the transactions that a user has entered
+        query (str): The query the user will ask against said embedding
+
+    Returns:
+        str: Response to query
+    """
+    try:
+        openai_api_key = os.environ['OPENAI_API_KEY']
+
+        embeddings = OpenAIEmbeddings(
+            model="text-embedding-3-small",
+            openai_api_key=openai_api_key
+        )
+
+        print(f"answer_transactions_query: \n document_splits: {type(document_splits)}\n")
+
+        llm = ChatOpenAI(
+            openai_api_key=openai_api_key,
+            model_name="gpt-3.5-turbo",
+            temperature=0.0
+        )
+
+        prompt_template = """
+            Use the following pieces of context to answer the question at the end.
+            If you don't know the answer, please think rationally answer from your own knowledge base
+
+            {context}
+
+            Question: {question}
+        """
+        prompt = PromptTemplate(
+            template=prompt_template, input_variables=["context", "question"]
+        )
+
+        chain_type_kwargs = {"prompt": prompt}
+
+        vectorestore = Chroma.from_documents(documents=document_splits,embedding=embeddings)
+
+        qa = RetrievalQA.from_chain_type(
+            llm=llm,
+            retriever=vectorestore.as_retriever(),
+            chain_type_kwargs=chain_type_kwargs
+        )
+
+        print(f"answer_transactions_query: \n qa: {qa}\n")
+
+        answer = qa({"query": query})
+
+        print(f"answer_transactions_query: \n answer: {answer['result']}\n")
+        
+        result_split = answer['result'].split("\n")
+        result = ''.join(item for item in result_split)
+
+        return result
+    
+    except Exception as e:
+        raise HTTPException(status_code = 500, detail=f"answer_query error: {str(e)}")
+    
+
+        # Note: Attempted to use Pinecone however it failed to learn from documents so settled for Chroma
+
+        # batch_limit = 20
+
+        # pinecone_api_key = os.environ['PINECONE_API_KEY']
+
+        # namespace = "transactionsvector"
+        # index_name = "transactionsrag"
+
+        # pc = Pinecone(api_key=pinecone_api_key)
+        #  if index_name in pc.list_indexes().names():
+        #     pc.delete_index(index_name)
+
+        # pc.create_index(
+        #     name=index_name,
+        #     dimension=1536,
+        #     metric="cosine",
+        #     spec=ServerlessSpec(
+        #         cloud="aws", region="us-east-1"
+        #     )
+        # )
+
+        # index = pc.Index(index_name)
+        # transactions_search = PineconeVectorStore.from_documents(
+        #     documents=document_splits,
+        #     index_name=index_name,
+        #     embedding=embeddings,
+        #     namespace=namespace
+        # )
+
+        # print(f"answer_transactions_query: \n transactions_search: {transactions_search}\n")
+
+        # for ids in index.list(namespace=namespace):
+        #     query_item = index.query(
+        #         id=ids[0], 
+        #         namespace=namespace, 
+        #         top_k=1,
+        #         include_values=True,
+        #         include_metadata=True
+        #     )
+        #     print(f"answer_transactions_query: \n query_item: {query_item}\n")
+
+        # qa = RetrievalQA.from_chain_type(
+        #     llm=llm,
+        #     chain_type="stuff",
+        #     retriever=transactions_search.as_retriever(),
+        #     chain_type_kwargs=chain_type_kwargs
+        # )
+

app/service/transactions_query_rag.py

@@ -1,116 +0,0 @@
-from pinecone import Pinecone, ServerlessSpec
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_openai import OpenAIEmbeddings
-from langchain_pinecone import PineconeVectorStore
-from langchain_openai import ChatOpenAI
-from langchain.chains import RetrievalQA
-
-from fastapi import HTTPException
-
-
-import os
-import pandas as pd
-from uuid import uuid4
-
-async def answer_query(df: pd.DataFrame, query: str) -> str:
-    """Creates an embedding of the transactions table and then returns the answer for the given query.
-    Args:
-        df (pd.DataFrame): DataFrame containing the transactions that a user has entered
-        query (str): The query the user will ask against said embedding
-
-    Returns:
-        str: Response to query
-    """
-    try:
-        batch_limit = 100
-        pinecone_api_key = os.environ['PINECONE_API_KEY']
-        openai_api_key = os.environ['OPENAI_API_KEY']
-        namespace = "transactionsvector"
-
-        pc = Pinecone(api_key=pinecone_api_key)
-
-        embeddings = OpenAIEmbeddings(
-            model="text-embedding-3-small",
-            openai_api_key=openai_api_key
-        )
-
-        index_name = "transactions_rag"
-
-        if index_name in pc.list_indexes().names():
-            pc.delete_index(index_name)
-
-        pc.create_index(
-            name=index_name,
-            dimension=1536,
-            metric="cosine",
-            spec=ServerlessSpec(
-                cloud="aws",
-                region="us-east-1"
-            )
-        )
-
-        index = pc.Index(index_name)
-
-        texts = []
-        all_texts = []
-        metadatas = []
-
-        text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=1000,
-            chunk_overlap=100
-        )
-
-        for _, record in df.iterrows():
-            content_texts = text_splitter.split_text(record['content'])
-            
-            metadata = {
-                'user_id': str(record['user_id'])
-            }
-            content_metadata = [{
-                "chunk": j, "text": text, **metadata
-            } for j, text in enumerate(content_texts)]
-
-            texts.extend(content_texts)
-            all_texts.extend(content_texts)
-            metadatas.extend(content_metadata)
-
-            # If we have reached the batch limit, then add the texts and reset
-            if len(texts) >= batch_limit:
-                ids = [str(uuid4()) for _ in range(len(texts))]
-                embeds = embeddings.embed_documents(texts)
-                index.upsert(vectors=zip(ids, embeds, metadatas))
-                texts = []
-                metadatas = []
-
-        if len(texts) > 0:
-            ids = [str(uuid4()) for _ in range(len(texts))]
-            embeds = embeddings.embed_documents(texts)
-            index.upsert(vectors=zip(ids, embeds, metadatas))
-
-        
-
-        transactions_search = PineconeVectorStore.from_documents(
-            documents=all_texts,
-            index_name=index_name,
-            embedding=embeddings,
-            namespace=namespace
-        )
-
-        llm = ChatOpenAI(
-            openai_api_key=openai_api_key,
-            model_name="gpt-3.5-turbo",
-            temperature=0.0
-        )
-
-        qa = RetrievalQA.from_llm(
-            llm=llm,
-            retriever=transactions_search.as_retriever()
-        )
-
-        answer = qa.invoke(query)
-
-        return answer
-    
-    except Exception as e:
-        raise HTTPException(status_code = 500, detail=f"fetch_pinecone_service error: {str(e)}")
-

tests/utils.py

@@ -8,80 +8,240 @@ def get_fake_transactions(user_id: int) -> List[TransactionCreate]:
         TransactionCreate(
             user_id=user_id,
             transaction_date=datetime(2022, 1, 1),
-            category="category1",
-            name_description="name_description",
-            amount=1.0,
+            category="shopping",
+            name_description="Amazon",
+            amount=11.0,
             type=TransactionType.EXPENSE,
         ),
         TransactionCreate(
             user_id=user_id,
             transaction_date=datetime(2022, 1, 2),
-            category="category2",
-            name_description="name_description",
-            amount=2.0,
+            category="Streaming",
+            name_description="Netflix",
+            amount=4.0,
+            type=TransactionType.EXPENSE,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 3),
+            category="Other",
+            name_description="Consulting",
+            amount=3.0,
+            type=TransactionType.INCOME,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 4),
+            category="Other",
+            name_description="Selling Paintings",
+            amount=4.0,
+            type=TransactionType.INCOME,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 5),
+            category="Gym",
+            name_description="Gym membership",
+            amount=5.0,
+            type=TransactionType.EXPENSE,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 6),
+            category="Transportation",
+            name_description="Uber Taxi",
+            amount=6.0,
+            type=TransactionType.EXPENSE,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 7),
+            category="Other",
+            name_description="Freelancing",
+            amount=7.0,
+            type=TransactionType.INCOME,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 8),
+            category="Income",
+            name_description="Salary",
+            amount=8.0,
+            type=TransactionType.INCOME,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 9),
+            category="Shopping",
+            name_description="Amazon Lux",
+            amount=9.0,
+            type=TransactionType.EXPENSE,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 10),
+            category="Taxes",
+            name_description="CA Property Tax",
+            amount=10.0,
+            type=TransactionType.EXPENSE,
+        ),
+         TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 1),
+            category="shopping",
+            name_description="Amazon",
+            amount=11.0,
+            type=TransactionType.EXPENSE,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 2),
+            category="Streaming",
+            name_description="Netflix",
+            amount=4.0,
+            type=TransactionType.EXPENSE,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 3),
+            category="Other",
+            name_description="Consulting",
+            amount=3.0,
+            type=TransactionType.INCOME,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 4),
+            category="Other",
+            name_description="Selling Paintings",
+            amount=4.0,
+            type=TransactionType.INCOME,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 5),
+            category="Gym",
+            name_description="Gym membership",
+            amount=5.0,
+            type=TransactionType.EXPENSE,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 6),
+            category="Transportation",
+            name_description="Uber Taxi",
+            amount=6.0,
+            type=TransactionType.EXPENSE,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 7),
+            category="Other",
+            name_description="Freelancing",
+            amount=7.0,
+            type=TransactionType.INCOME,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 8),
+            category="Income",
+            name_description="Salary",
+            amount=8.0,
+            type=TransactionType.INCOME,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 9),
+            category="Shopping",
+            name_description="Amazon Lux",
+            amount=9.0,
+            type=TransactionType.EXPENSE,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 10),
+            category="Taxes",
+            name_description="CA Property Tax",
+            amount=10.0,
+            type=TransactionType.EXPENSE,
+        ),
+         TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 1),
+            category="shopping",
+            name_description="Amazon",
+            amount=11.0,
+            type=TransactionType.EXPENSE,
+        ),
+        TransactionCreate(
+            user_id=user_id,
+            transaction_date=datetime(2022, 1, 2),
+            category="Streaming",
+            name_description="Netflix",
+            amount=4.0,
             type=TransactionType.EXPENSE,
         ),
         TransactionCreate(
             user_id=user_id,
             transaction_date=datetime(2022, 1, 3),
-            category="category3",
-            name_description="name_description",
+            category="Other",
+            name_description="Consulting",
             amount=3.0,
             type=TransactionType.INCOME,
         ),
         TransactionCreate(
             user_id=user_id,
             transaction_date=datetime(2022, 1, 4),
-            category="category1",
-            name_description="name_description",
+            category="Other",
+            name_description="Selling Paintings",
             amount=4.0,
             type=TransactionType.INCOME,
         ),
         TransactionCreate(
             user_id=user_id,
             transaction_date=datetime(2022, 1, 5),
-            category="category2",
-            name_description="name_description",
+            category="Gym",
+            name_description="Gym membership",
             amount=5.0,
             type=TransactionType.EXPENSE,
         ),
         TransactionCreate(
             user_id=user_id,
             transaction_date=datetime(2022, 1, 6),
-            category="category3",
-            name_description="name_description",
+            category="Transportation",
+            name_description="Uber Taxi",
             amount=6.0,
             type=TransactionType.EXPENSE,
         ),
         TransactionCreate(
             user_id=user_id,
             transaction_date=datetime(2022, 1, 7),
-            category="category1",
-            name_description="name_description",
+            category="Other",
+            name_description="Freelancing",
             amount=7.0,
             type=TransactionType.INCOME,
         ),
         TransactionCreate(
             user_id=user_id,
             transaction_date=datetime(2022, 1, 8),
-            category="category2",
-            name_description="name_description",
+            category="Income",
+            name_description="Salary",
             amount=8.0,
             type=TransactionType.INCOME,
         ),
         TransactionCreate(
             user_id=user_id,
             transaction_date=datetime(2022, 1, 9),
-            category="category3",
-            name_description="name_description",
+            category="Shopping",
+            name_description="Amazon Lux",
             amount=9.0,
             type=TransactionType.EXPENSE,
         ),
         TransactionCreate(
             user_id=user_id,
             transaction_date=datetime(2022, 1, 10),
-            category="category1",
-            name_description="name_description",
+            category="Taxes",
+            name_description="CA Property Tax",
             amount=10.0,
             type=TransactionType.EXPENSE,
         ),