Add categorizer files that were not committed

app/categorization/categorizer.py

@@ -0,0 +1,140 @@
+# Standard library imports
+import re
+import ast
+import json
+import logging
+from typing import Any, List, Tuple, Optional, Dict, Union
+
+# Third-party library imports
+import numpy as np
+import pandas as pd
+import asyncio
+from rapidfuzz import process
+from tenacity import retry, wait_random_exponential, stop_after_attempt
+from pydantic import ValidationError
+
+# Local application/library specific imports
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import LLMChain
+from langchain.output_parsers import PydanticOutputParser, OutputFixingParser
+from langchain.prompts import PromptTemplate
+import template as template
+from config import CATEGORY_REFERENCE_OUTPUT_FILE, TX_PER_LLM_RUN
+
+
+def fuzzy_match_list_categorizer(
+    description: str,
+    descriptions: np.ndarray,
+    description_category_pairs: pd.DataFrame,
+    threshold: int = 75,
+) -> Optional[str]:
+    """Find the most similar transaction description and return its category.
+
+    This function uses fuzzy string matching to compare the input description
+    against a list of known descriptions. If a sufficient match is found,
+    the function returns the category associated with the matched description.
+
+    Args:
+        description (str): The transaction description to categorize.
+        descriptions (np.ndarray): Known descriptions to compare against.
+        description_category_pairs (pd.DataFrame): DataFrame mapping descriptions to categories.
+        threshold (int): Minimum similarity score to consider a match.
+
+    Returns:
+        str or None: Category of the matched description, or None if no match found.
+    """
+
+    # Fuzzy-match this description against the reference descriptions
+    match_results = process.extractOne(description, descriptions, score_cutoff=threshold)
+
+    # If a match is found, return the category of the matched description
+    if match_results:
+        return description_category_pairs.at[match_results[2], 'category']
+    
+    return None
+
+
+async def llm_list_categorizer(tx_list: pd.DataFrame) -> pd.DataFrame:
+    """Categorize a list of transactions using a language model.
+
+    This function uses a Language Model (LLM) to categorize a list of transaction descriptions.
+    It splits the input DataFrame into chunks and processes each chunk asynchronously to improve performance.
+
+    Args:
+        tx_list (pd.DataFrame): DataFrame containing the transaction descriptions to categorize.
+
+    Returns:
+        pd.DataFrame: DataFrame mapping transaction descriptions to their inferred categories.
+    """
+
+    # Initialize language model and prompt
+    llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo-0125")
+    prompt = PromptTemplate.from_template(template=template.CATEGORY_TEMPLATE)    
+    chain = LLMChain(llm=llm, prompt=prompt)
+
+    # Iterate over the DataFrame in batches of TX_PER_LLM_RUN transactions
+    tasks = [llm_sublist_categorizer(tx_list.attrs['file_name'], chain=chain, tx_descriptions="\n".join(chunk['description']).strip())
+             for chunk in np.array_split(tx_list, tx_list.shape[0] // TX_PER_LLM_RUN + 1)]
+
+    # Gather results and extract (valid) outputs
+    # The results variable is a list of 'results', each 'result' being the output of a single LLM run
+    results = await asyncio.gather(*tasks)
+
+    # Extract valid results (each valid result is a list of description-category pairs)
+    valid_results = [result['output'] for result in results if result['valid']]
+
+    # Flatten the list of valid results to obtain a single list of description-category pairs
+    valid_outputs = [output for valid_result in valid_results for output in valid_result]
+
+    # Return a DataFrame with the valid outputs
+    return pd.DataFrame(valid_outputs, columns=['description', 'category'])
+
+
+@retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6))
+async def llm_sublist_categorizer(
+    file_name: str,
+    chain: LLMChain,
+    tx_descriptions: str,
+) -> Dict[str, Union[bool, List[Tuple[str, str]]]]:
+    """Categorize a batch of transactions using a language model.
+
+    This function takes a batch of transaction descriptions and passes them to a language model
+    for categorization. The function retries on failure, with an exponential backoff.
+
+    Args:
+        file_name (str): Name of the file the transaction descriptions were extracted from.
+        chain (LLMChain): Language model chain to use for categorization.
+        tx_descriptions (str): Concatenated transaction descriptions to categorize.
+
+    Returns:
+        dict: Dictionary containing a 'valid' flag and a list of categorized descriptions.
+    """
+
+    raw_result = await chain.arun(input_data=tx_descriptions)
+
+    logger = logging.getLogger(__name__)
+    result = {'valid': True, 'output': []}
+    try:
+        # Create a pattern to match a list Description-Category pairs (List[Tuple[str, str]])
+        pattern = r"\['([^']+)', '([^']+)'\]"
+        
+        # Use it to extract all the correctly formatted pairs from the raw result
+        matches = re.findall(pattern, raw_result.replace("\\'", "'"))
+
+        # Loop over the matches, and try to parse them to ensure the content is valid
+        valid_outputs = []
+        for match in matches:
+            try:
+                parsed_pair = ast.literal_eval(str(list(match)))
+                valid_outputs.append(parsed_pair)
+            except Exception as e:
+                logger.log(logging.ERROR, f"Parsing Error: {e}\nMatch: {match}\n")
+                result['valid'] = False
+
+        result['output'] = valid_outputs
+
+    except Exception as e:
+        logging.log(logging.ERROR, f"| File: {file_name} | Unexpected Error: {e}\nRaw Result: {raw_result}")
+        result['valid'] = False
+    
+    return result

app/categorization/categorizer_list.py

@@ -0,0 +1,70 @@
+# Standard library imports
+import os
+from datetime import datetime
+
+# Third-party library imports
+import pandas as pd
+
+# Local application/library specific imports
+from config import CATEGORY_REFERENCE_OUTPUT_FILE
+from categorizer import llm_list_categorizer, fuzzy_match_list_categorizer
+
+
+async def categorize_list(tx_list: pd.DataFrame) -> pd.DataFrame:
+    """Asynchronously categorize a list of transactions.
+
+    This function categorizes a list of transactions using a combination of fuzzy matching
+    and a language model. It looks up new transaction descriptions in the reference file
+    (a combination of user input and previous executions) to minimize API calls.
+    Any uncategorized transactions are sent to the language model, and new description-category
+    pairs are added to the reference file.
+
+    Args:
+        tx_list (pd.DataFrame): The list of transactions to categorize.
+
+    Returns:
+        pd.DataFrame: The original DataFrame with an additional column for the category.
+    """
+
+    if os.path.exists(CATEGORY_REFERENCE_OUTPUT_FILE):
+        # Read description-category pairs from the reference file
+        description_category_pairs = pd.read_csv(
+            CATEGORY_REFERENCE_OUTPUT_FILE, header=None, names=['description', 'category']
+        )
+
+        # Extract only descriptions for faster matching
+        descriptions = description_category_pairs['description'].values
+
+        # Use fuzzy matching to find similar descriptions and assign the category
+        tx_list['category'] = tx_list['description'].apply(
+            fuzzy_match_list_categorizer,
+            args=(descriptions, description_category_pairs),
+        )
+
+    # Filter out uncategorized transactions, deduplicate, and sort by description
+    uncategorized_descriptions = (
+        tx_list[tx_list['category'].isnull()]
+        .drop_duplicates(subset=['description'])
+        .sort_values(by=['description'])
+    )
+
+    # Ask the language model to categorize the remaining descriptions
+    if not uncategorized_descriptions.empty:
+        categorized_descriptions = await llm_list_categorizer(
+            uncategorized_descriptions[['description', 'category']]
+        )
+
+        categorized_descriptions.dropna(inplace=True)
+
+        # Update the category for uncategorized transactions based on the language model results
+        if not categorized_descriptions.empty:
+            tx_list['category'] = tx_list['category'].fillna(
+                tx_list['description'].map(
+                    categorized_descriptions.set_index('description')['category']
+                )
+            )
+        
+        # Fill remaining NaN values in 'category' with 'Other'
+        tx_list['category'] = tx_list['category'].fillna('Other')
+
+    return tx_list

app/categorization/config.py

@@ -0,0 +1,6 @@
+# Data Folders
+RESULT_OUTPUT_FILE = 'data/tx_data/output/result_master_data.csv'
+CATEGORY_REFERENCE_OUTPUT_FILE = 'data/ref_data/category_ref_master_data.csv'
+
+# LLM CONFIG
+TX_PER_LLM_RUN = 10

app/categorization/file_processing.py

@@ -10,7 +10,7 @@ import pandas as pd
 from dateparser import parse
 
 from categorizer_list import categorize_list
-from config import TEXT_OUTPUT_FILE, CATEGORY_REFERENCE_OUTPUT_FILE
+from config import RESULT_OUTPUT_FILE, CATEGORY_REFERENCE_OUTPUT_FILE
 
 # Read file and process it (e.g. categorize transactions)
 async def process_file(file_path: str) -> Dict[str, Union[str, pd.DataFrame]]:
@@ -100,7 +100,7 @@ def save_results(results: List) -> None:
             error_messages.append(f"{result['file_name']}: {result['error']}")  
 
     # Write contents to output file (based on file type)
-    tx_list.to_csv(TEXT_OUTPUT_FILE, mode="a", index=False, header=not os.path.exists(TEXT_OUTPUT_FILE))
+    tx_list.to_csv(RESULT_OUTPUT_FILE, mode="a", index=False, header=not os.path.exists(RESULT_OUTPUT_FILE))
 
     new_ref_data = tx_list[['name/description', 'category']]
     if os.path.exists(CATEGORY_REFERENCE_OUTPUT_FILE):

app/categorization/template.py

@@ -0,0 +1,56 @@
+CATEGORY_TEMPLATE = """
+    <context>
+    Your task is to create a list of [description, category] lists, where in each list item:
+    - the description is exactly the same you receive as input
+    - the category of the transaction (choose one from the list below based on the description)
+    </context>
+    
+    
+    <categories>
+    The following list contains the categories and associated keywords you often see in transaction descriptions:
+    -ATM: atm, cash, withdraw
+    -Auto: auto body
+    -Bars: bar, pubs, irish, brewery
+    -Beauty: body
+    -Cashback: cashback, reward, bonus, cash back
+    -Clothing: clothing, shoes, accessories
+    -Coffee Shops: coffee, cafe, tea, Starbucks, Dunkin
+    -Credit Card Payment: card payment, autopay
+    -Education: kindle, tuition
+    -Entertainment: event, show, movies, cinema, theater
+    -Fees: Fee
+    -Food: snack, Donalds, Burger King, KFC, Subway, Pizza, Domino, Taco Bell, Wendy, Chick-fil-A, Popeyes, Arby's, Chipotle
+    -Fuel: fuel, gas, petrol
+    -Gifts: donation, gift
+    -Groceries: groceries, supermarket, food, familia
+    -Gym: gym, fitness, yoga, pilates, crossfit
+    -Home: Ikea
+    -Housing: rent, mortgage
+    -Income: refund, deposit, paycheck
+    -Insurance: insurance
+    -Medical: medical, doctor, dentist, hospital, clinic
+    -Pets: vet, veterinary, pet, dog, cat
+    -Pharmacy: pharmacy, drugstore, cvs, walgreens, rite aid, duane
+    -Restaurants: restaurant, lunch, dinner
+    -Services: service, laundry, dry cleaning
+    -Shopping: shopping, amazon, walmart, target, safeway
+    -Streaming: Netflix, Spotify, Hulu, HBO
+    -Taxes: tax, irs
+    -Technology: technology, software, hardware, electronics
+    -Transportation: bus, train, subway, metro, airline, uber, lyft, taxi
+    -Travel: travel, holiday, trip, airbnb, kiwi, hotel, hostel, resort, kiwi, kayak, expedia, booking.com
+    -Transfer: payment from
+    -Utilities: electricity, water, gas, ting, verizon, comcast, sprint, t-mobile, at&t, mint
+    -Other: use this when very uncertain about the category
+    </categories>
+
+    <formatting_instructions>
+    Your output should be a valid list of lists (e.g. [[description1, category1], [description2, category2], ...]] parse-able by the command ast.literal_eval(output)
+    Don't include any kind of commentary, return carriages, spaces or other characters in the output
+    Fill all categories; if you don't know which one to choose, choose 'Other'
+    </formatting_instructions>
+    
+    
+    <financial_transactions>
+    {input_data}
+    </financial_transactions>"""

app/transactions_rag/categorize_transactions.ipynb

@@ -0,0 +1,201 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Defaulting to user installation because normal site-packages is not writeable\n",
+      "Requirement already satisfied: pandas in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (2.0.3)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pandas) (2.9.0.post0)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pandas) (2024.1)\n",
+      "Requirement already satisfied: tzdata>=2022.1 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pandas) (2024.1)\n",
+      "Requirement already satisfied: numpy>=1.20.3 in /Users/patrickalexis/Library/Python/3.8/lib/python/site-packages (from pandas) (1.24.4)\n",
+      "Requirement already satisfied: six>=1.5 in /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas) (1.15.0)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Date</th>\n",
+       "      <th>Name / Description</th>\n",
+       "      <th>Expense/Income</th>\n",
+       "      <th>Amount</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2023-12-30</td>\n",
+       "      <td>Comcast Internet</td>\n",
+       "      <td>Expense</td>\n",
+       "      <td>9.96</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2023-12-30</td>\n",
+       "      <td>Lemonade Home Insurance</td>\n",
+       "      <td>Expense</td>\n",
+       "      <td>17.53</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2023-12-30</td>\n",
+       "      <td>Monthly Appartment Rent</td>\n",
+       "      <td>Expense</td>\n",
+       "      <td>2000.00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2023-12-30</td>\n",
+       "      <td>Staples Office Supplies</td>\n",
+       "      <td>Expense</td>\n",
+       "      <td>12.46</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>2023-12-29</td>\n",
+       "      <td>Selling Paintings</td>\n",
+       "      <td>Income</td>\n",
+       "      <td>13.63</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "         Date       Name / Description Expense/Income   Amount\n",
+       "0  2023-12-30         Comcast Internet        Expense     9.96\n",
+       "1  2023-12-30  Lemonade Home Insurance        Expense    17.53\n",
+       "2  2023-12-30  Monthly Appartment Rent        Expense  2000.00\n",
+       "3  2023-12-30  Staples Office Supplies        Expense    12.46\n",
+       "4  2023-12-29        Selling Paintings         Income    13.63"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Read the transactions csv\n",
+    "!pip3 install pandas\n",
+    "\n",
+    "import pandas as pd\n",
+    "df = pd.read_csv(\"transactions_2024.csv\")\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array(['Lemonade Home Insurance', 'Monthly Appartment Rent',\n",
+       "       'Staples Office Supplies', 'Selling Paintings', 'Spotify',\n",
+       "       'Target', 'IT Consulting', 'Phone', 'ML Consulting'], dtype=object)"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Get Unique transactions in the Name/Description column\n",
+    "unique_transactions = df[\"Name / Description\"].unique()\n",
+    "unique_transactions[1:10]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'categorization'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Process the transactions csv and get the results from the categorization llm utility\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcategorization\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mfile_processing\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m process_file, save_results\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdotenv\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m load_dotenv\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01masyncio\u001b[39;00m\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'categorization'"
+     ]
+    }
+   ],
+   "source": [
+    "# Process the transactions csv and get the results from the categorization llm utility\n",
+    "from categorization.file_processing import process_file, save_results\n",
+    "from dotenv import load_dotenv\n",
+    "import asyncio\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "async def apply_categorization():\n",
+    "    processed_file = process_file(\"transactions_2024.csv\")\n",
+    "\n",
+    "    print(\"\\nProcessing file\")\n",
+    "    result = await asyncio.gather(processed_file)\n",
+    "\n",
+    "    save_results(results)\n",
+    "    print(results)\n",
+    "    \n",
+    "\n",
+    "asyncio.run(apply_categorization())"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.8.9 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.9"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

app/transactions_rag/transactions_2024.csv

@@ -25,5 +25,5 @@ Date,Name / Description,Expense/Income,Amount
 2022-10-22,Apple Services,Expense,41.25
 2022-10-21,Netflix,Expense,22.8
 2022-01-16,Amazon Lux,Expense,24.11
-2022-01-15,Classpass* Monthly,Expense,30.08
-2022-01-14,Amazon,Expense,11.0
+2022-01-15,Burger King,Expense,30.08
+2022-01-14,Amazon,Expense,11.0

requirements.txt

@@ -8,4 +8,14 @@ llama-index-vector-stores-pinecone==0.1.3
 llama-index==0.10.28
 python-dotenv==1.0.0
 traceloop-sdk==0.15.11
-uvicorn==0.23.2
+uvicorn==0.23.2
+
+langchain
+python-dotenv
+openai
+tenacity
+rapidfuzz
+pydantic
+dateparser
+pandas
+path