Fixed code in file_processing file although date formatting needs tweaking

app/api/routers/file_upload.py

@@ -1,5 +1,5 @@
 from typing import Annotated
-from fastapi import APIRouter, UploadFile, Depends
+from fastapi import APIRouter, UploadFile, Depends, HTTPException
 from app.categorization.file_processing import process_file, save_results
 from app.schema.index import FileUploadCreate
 import asyncio
@@ -9,6 +9,7 @@ import csv
 from app.engine.postgresdb import get_db_session
 from sqlalchemy.ext.asyncio import AsyncSession
 
+
 file_upload_router = r = APIRouter(prefix="/api/v1/file_upload", tags=["file_upload"])
 
 @r.post(
@@ -21,23 +22,45 @@ file_upload_router = r = APIRouter(prefix="/api/v1/file_upload", tags=["file_upl
 )
 async def create_file(input_file: UploadFile, db: AsyncSession = Depends(get_db_session)):
     try:
+        result = {"file_name": input_file.filename, "output": "", "result": "", "processed_file": "", "error": ""}
+
          # Create directory to store all uploaded .csv files
         file_upload_directory_path = "data/tx_data/input"
+        output_csv_file_path = os.path.join(file_upload_directory_path, input_file.filename)
         if not os.path.exists(file_upload_directory_path):
             os.makedirs(file_upload_directory_path)
 
-        # Write items of .csv filte to directory
-        with open(os.path.join(file_upload_directory_path, input_file.filename)) as output_file:
-            [output_file.write(" ".join(row)+'\n') for row in csv.reader(input_file)]
-        output_file.close()
+        input_csv_file = open(output_csv_file_path, "a")
+
+        try:
+            if input_file.filename.endswith(".csv"):
+                read_csv = await input_file.read()
+                # result["output"] = read_csv
+
+                # Write items of .csv filte to directory
+                with open(output_csv_file_path, 'w') as csv_file:
+                    result["processed_file"] = read_csv
+                    [csv_file.write(" ".join(row)+'\n') for row in read_csv.decode("utf-8").splitlines()]
+                csv_file.close()
+                
+                # with open(output_csv_file_path, 'r') as csv_file:
+                #     result["output"] = csv_file.read()
+                # csv_file.close()
+                
+                # print(f"create_file result: {result}")
 
-        # With the newly created file and it's path, process and save it for embedding
-        processed_file = process_file(os.path.realpath(input_file.filename))
-        result = await asyncio.gather(processed_file)
-        await save_results(db, result)
+                # With the newly created file and it's path, process and save it for embedding
+                processed_file = process_file(output_csv_file_path)
+                result["processed_file"] = processed_file
+                result = await asyncio.gather(processed_file)
+                result["result"] = result
+                await save_results(db, result)
+                   
+        except Exception as e:
+            result["error"] = str(e)
+            raise HTTPException(status_code = 500, detail=f"create_file inner exception: {result} \n")
 
-    except Exception:
-        return {"message": "There was an error uploading this file. Ensure you have a .csv file with the following columns:"
-        "\n transaction_date, type, category, name_description, amount"}
+    except Exception as e:
+      raise HTTPException(status_code = 500, detail=f"create_file outer exception: {result} \n")
 
     return {"message": f"Successfully uploaded {input_file.filename}"}

app/categorization/categorizer_list.py

@@ -29,14 +29,14 @@ async def categorize_list(tx_list: pd.DataFrame) -> pd.DataFrame:
     if os.path.exists(CATEGORY_REFERENCE_OUTPUT_FILE):
         # Read description-category pairs from the reference file
         description_category_pairs = pd.read_csv(
-            CATEGORY_REFERENCE_OUTPUT_FILE, header=None, names=['name/description', 'category']
+            CATEGORY_REFERENCE_OUTPUT_FILE, header=None, names=['name_description', 'category']
         )
 
         # Extract only descriptions for faster matching
-        descriptions = description_category_pairs['name/description'].values
+        descriptions = description_category_pairs['name_description'].values
 
         # Use fuzzy matching to find similar descriptions and assign the category
-        tx_list['category'] = tx_list['name/description'].apply(
+        tx_list['category'] = tx_list['name_description'].apply(
             fuzzy_match_list_categorizer,
             args=(descriptions, description_category_pairs),
         )
@@ -44,14 +44,14 @@ async def categorize_list(tx_list: pd.DataFrame) -> pd.DataFrame:
     # Filter out uncategorized transactions, deduplicate, and sort by description
     uncategorized_descriptions = (
         tx_list[tx_list['category'].isnull()]
-        .drop_duplicates(subset=['name/description'])
-        .sort_values(by=['name/description'])
+        .drop_duplicates(subset=['name_description'])
+        .sort_values(by=['name_description'])
     )
 
     # Ask the language model to categorize the remaining descriptions
     if not uncategorized_descriptions.empty:
         categorized_descriptions = await llm_list_categorizer(
-            uncategorized_descriptions[['name/description', 'category']]
+            uncategorized_descriptions[['name_description', 'category']]
         )
 
         categorized_descriptions.dropna(inplace=True)
@@ -59,8 +59,8 @@ async def categorize_list(tx_list: pd.DataFrame) -> pd.DataFrame:
         # Update the category for uncategorized transactions based on the language model results
         if not categorized_descriptions.empty:
             tx_list['category'] = tx_list['category'].fillna(
-                tx_list['name/description'].map(
-                    categorized_descriptions.set_index('name/description')['category']
+                tx_list['name_description'].map(
+                    categorized_descriptions.set_index('name_description')['category']
                 )
             )
         

app/categorization/file_processing.py

@@ -8,6 +8,8 @@ from datetime import datetime
 
 import pandas as pd
 from dateparser import parse
+from fastapi import  HTTPException
+import csv
 
 from app.categorization.categorizer_list import categorize_list
 from app.categorization.config import RESULT_OUTPUT_FILE, CATEGORY_REFERENCE_OUTPUT_FILE
@@ -42,8 +44,9 @@ async def process_file(file_path: str) -> Dict[str, Union[str, pd.DataFrame]]:
     except Exception as e:
         # Return an error indicator and exception info
         logging.debug(logging.ERROR, f"| File: {file_name} | Unexpected Error: {e}")
-        print(f"ERROR processing file {file_name}: {e}")
+        # print(f"ERROR processing file {file_name}: {e}")
         result["error"] = str(e)
+        raise HTTPException(status_code = 500, detail=f"process_file result: {result}")
 
     return result
 
@@ -58,23 +61,42 @@ def standardize_csv_file(file_path: str) -> pd.DataFrame:
     Returns:
         pd.DataFrame: Prepared transaction data.
     """
+    result = {"csv_file": "", "file_path": file_path, "tx_list_columns": {}, "transaction_date": "", "tx_list_new_columns": "", "error": ""}
     try:
-        tx_list = pd.read_csv(file_path, index_col=False)
-        tx_list.attrs["file_name"] = file_path
-        tx_list.columns = tx_list.columns.str.lower().str.strip()
-
-        # Standardize dates to YYYY/MM/DD format
-        tx_list["date"] = pd.to_datetime(tx_list["date"]).dt.strftime("%Y/%m/%d")
-
-        # Add source and reindex to desired tx format; category column is new and therefore empty
-        tx_list.loc[:, "source"] = os.path.basename(file_path)
-        tx_list = tx_list.reindex(columns=["transaction_date", "type", "category", "name_description", "amount"])
+        with open(file_path, 'r') as csv_file:
+            # result["csv_file"] = csv_file.read()
+            # reader = csv.reader(csv_file)
+            # data = list(reader)
+            # tx_list = pd.DataFrame(data, columns=data[0])
+
+            # result["csv_file"] = csv_file.read()
+            tx_list = pd.read_csv(file_path, parse_dates=['transaction_date'])
+            # tx_list.dropna(inplace=True)
+
+            # result["tx_list_columns"] = tx_list
+            tx_list.attrs["file_name"] = file_path
+            tx_list.columns = tx_list.columns.str.lower().str.strip()
+        
+            # Standardize dates to YYYY/MM/DD format
+            result["transaction_date"] = pd.to_datetime(tx_list["transaction_date"])
+            print(f"transaction_date: {tx_list['transaction_date']}")
+            tx_list["transaction_date"] = pd.to_datetime(tx_list["transaction_date"]).dt.strftime("%Y/%m/%d")
+
+            # Add source and reindex to desired tx format; category column is new and therefore empty
+            tx_list.loc[:, "source"] = os.path.basename(file_path)
+            tx_list = tx_list.reindex(columns=["transaction_date", "type", "category", "name_description", "amount"])
+            # result["tx_list_new_columns"] = tx_list
+            
+        return tx_list
     
     except Exception as e:
         # Return an error indicator and exception info
+        # print(f"standardize_csv_file exception: {e}")
         logging.debug("standardize_csv_file Error: {e}")
+        result["error"] = str(e)
+        raise HTTPException(status_code = 401, detail=f"standardize_csv_file result: {result}")
 
-    return tx_list
+    # return tx_list
 
 
 async def save_results(db: AsyncSession, results: List) -> None:
@@ -98,6 +120,7 @@ async def save_results(db: AsyncSession, results: List) -> None:
     col_list = ["transaction_date", "type", "category", "name_description", "amount"]
     tx_list = pd.DataFrame(columns=col_list)
     for result in results:
+        print(f"save_results result: {result}")
         if not result["error"]:
             ok_files.append(result["file_name"])
             result_df = result["output"]
@@ -113,6 +136,10 @@ async def save_results(db: AsyncSession, results: List) -> None:
     await Transaction.bulk_create(db, txn_list_to_save)
 
     new_ref_data = tx_list[["name_description", "category"]]
+
+    if not os.path.exists(CATEGORY_REFERENCE_OUTPUT_FILE):
+        os.makedirs(CATEGORY_REFERENCE_OUTPUT_FILE)
+
     if os.path.exists(CATEGORY_REFERENCE_OUTPUT_FILE):
         # If it exists, add master file to interim results
         old_ref_data = pd.read_csv(CATEGORY_REFERENCE_OUTPUT_FILE, names=["name_description", "category"], header=0)

tests/test_file_upload.py

@@ -0,0 +1,20 @@
+from datetime import datetime
+from typing import List
+from pathlib import Path
+from fastapi import Depends
+from fastapi.testclient import TestClient
+import pytest
+
+from app.model.transaction import Transaction
+from app.schema.index import TransactionType, TransactionCreate
+
+from sqlalchemy.ext.asyncio import AsyncSession
+from app.engine.postgresdb import get_db_session
+
+@pytest.mark.asyncio
+async def test_file_upload(client: TestClient, get_db_session_fixture: AsyncSession) -> None:
+    _test_upload_file = Path('/Users/patrickalexis/Documents/codepath-ai-course/codepath-group-project/backend/app/transactions_rag/transactions_2024.csv', 'new-index.json')
+    _files = {'input_file': _test_upload_file.open('rb')}
+
+    response = client.post(("/api/v1/file_upload/"),files=_files)
+    assert response.status_code == 201