Push experimentation code for file upload fix

app/categorization/file_processing.py

@@ -41,7 +41,7 @@ async def process_file(file_path: str) -> Dict[str, Union[str, pd.DataFrame]]:
 
     except Exception as e:
         # Return an error indicator and exception info
-        logging.log(logging.ERROR, f"| File: {file_name} | Unexpected Error: {e}")
+        logging.debug(logging.ERROR, f"| File: {file_name} | Unexpected Error: {e}")
         print(f"ERROR processing file {file_name}: {e}")
         result["error"] = str(e)
 
@@ -58,17 +58,21 @@ def standardize_csv_file(file_path: str) -> pd.DataFrame:
     Returns:
         pd.DataFrame: Prepared transaction data.
     """
+    try:
+        tx_list = pd.read_csv(file_path, index_col=False)
+        tx_list.attrs["file_name"] = file_path
+        tx_list.columns = tx_list.columns.str.lower().str.strip()
 
-    tx_list = pd.read_csv(file_path, index_col=False)
-    tx_list.attrs["file_name"] = file_path
-    tx_list.columns = tx_list.columns.str.lower().str.strip()
-
-    # Standardize dates to YYYY/MM/DD format
-    tx_list["date"] = pd.to_datetime(tx_list["date"]).dt.strftime("%Y/%m/%d")
+        # Standardize dates to YYYY/MM/DD format
+        tx_list["date"] = pd.to_datetime(tx_list["date"]).dt.strftime("%Y/%m/%d")
 
-    # Add source and reindex to desired tx format; category column is new and therefore empty
-    tx_list.loc[:, "source"] = os.path.basename(file_path)
-    tx_list = tx_list.reindex(columns=["date", "expense/income", "category", "name/description", "amount"])
+        # Add source and reindex to desired tx format; category column is new and therefore empty
+        tx_list.loc[:, "source"] = os.path.basename(file_path)
+        tx_list = tx_list.reindex(columns=["transaction_date", "type", "category", "name_description", "amount"])
+    
+    except Exception as e:
+        # Return an error indicator and exception info
+        logging.debug("standardize_csv_file Error: {e}")
 
     return tx_list
 
@@ -108,19 +112,20 @@ async def save_results(db: AsyncSession, results: List) -> None:
     txn_list_to_save = [TransactionCreate(**row.to_dict(), user_id=1) for _, row in tx_list.iterrows()]
     await Transaction.bulk_create(db, txn_list_to_save)
 
-    new_ref_data = tx_list[["name/description", "category"]]
+    new_ref_data = tx_list[["name_description", "category"]]
     if os.path.exists(CATEGORY_REFERENCE_OUTPUT_FILE):
         # If it exists, add master file to interim results
-        old_ref_data = pd.read_csv(CATEGORY_REFERENCE_OUTPUT_FILE, names=["name/description", "category"], header=0)
+        old_ref_data = pd.read_csv(CATEGORY_REFERENCE_OUTPUT_FILE, names=["name_description", "category"], header=0)
         new_ref_data = pd.concat([old_ref_data, new_ref_data], ignore_index=True)
 
     # Drop duplicates, sort, and write to create new Master File
-    new_ref_data.drop_duplicates(subset=["name/description"]).sort_values(by=["name/description"]).to_csv(
+    new_ref_data.drop_duplicates(subset=["name_description"]).sort_values(by=["name_description"]).to_csv(
         CATEGORY_REFERENCE_OUTPUT_FILE, mode="w", index=False, header=True
     )
 
     # Summarize results
     print(f"\nProcessed {len(results)} files: {len(ok_files)} successful, {len(ko_files)} with errors\n")
+    logging.debug(f"\nProcessed {len(results)} files: {len(ok_files)} successful, {len(ko_files)} with errors\n")
     if len(ko_files):
         print(f"Errors in the following files:")
         for message in error_messages:

app/settings.py

@@ -1,5 +1,6 @@
 import os
 from typing import Dict
+from dotenv import load_dotenv
 from llama_index.core.settings import Settings
 
 

app/transactions_rag/transactions_2024.csv

@@ -1,4 +1,4 @@
-Date,Name / Description,Expense/Income,Amount
+transaction_date,name_description,type,amount
 2023-12-30,Comcast Internet,Expense,9.96
 2023-12-30,Lemonade Home Insurance,Expense,17.53
 2023-12-30,Monthly Appartment Rent,Expense,2000.0