deploying code to hugging face

backend/.env.prod

@@ -0,0 +1,4 @@
+ENV=production
+HUGGINGFACE_API_TOKEN=your_production_api_token
+EMAIL_DIRECTORY_PATH=your_email_file_path
+

backend/.gitignore

@@ -0,0 +1,222 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Ignore installed dependencies
+/backend_env/
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+
+# See https://docs.github.com/get-started/getting-started-with-git/ignoring-files for more about ignoring files.
+
+# Compiled output
+/dist
+/tmp
+/out-tsc
+/bazel-out
+
+# Node
+/node_modules
+npm-debug.log
+yarn-error.log
+
+# IDEs and editors
+.idea/
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# Visual Studio Code
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+.history/*
+
+# Miscellaneous
+/.angular/cache
+.sass-cache/
+/connect.lock
+/coverage
+/libpeerconnection.log
+testem.log
+/typings
+
+# System files
+.DS_Store
+Thumbs.db
+

backend/Dockerfile

@@ -0,0 +1,17 @@
+# Use the official Python 3.11.5 image
+FROM python:3.11.5
+
+# Set the working directory
+WORKDIR /app
+
+# Copy the backend folder into the container
+COPY backend /app
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Expose the port FastAPI runs on
+EXPOSE 7860
+
+# Run FastAPI app
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

backend/README.md

@@ -0,0 +1,90 @@
+# 📧 Gen AI Email Processing
+
+This project is an AI-powered email processing system that reads, extracts, classifies, and analyzes emails and documents using LLaMA models. It supports OCR for images/PDFs and utilizes machine learning to classify emails.
+
+---
+
+## 🚀 Tech Stack
+- **Python** (Backend Development)
+- **FastAPI** (API Framework)
+- **PyTorch & Transformers** (AI Model - LLaMA)
+- **OCR Tools** (Tesseract, pdfplumber, python-docx)
+- **Langchain & Sentence-Transformers** (Text Processing & Embeddings)
+- **Uvicorn** (ASGI Server)
+- **Scikit-learn & Pandas** (Data Processing)
+- **LLama-CPP-Python** (LLaMA Model Integration)
+
+---
+
+## 📥 Installation & Setup
+Follow these steps to set up the project on your local machine:
+
+### 1️⃣ Clone the Repository
+```sh
+ git clone https://github.com/your-username/gen_ai_email_processing.git
+ cd gen_ai_email_processing
+```
+
+### 2️⃣ Create a Virtual Environment
+```sh
+ python -m venv env
+ source env/bin/activate  # On macOS/Linux
+ env\Scripts\activate    # On Windows
+```
+
+### 3️⃣ Install Dependencies
+```sh
+ pip install --upgrade pip
+ pip install -r requirements.txt
+```
+
+### 4️⃣ Run the FastAPI Server
+```sh
+ uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
+```
+
+### 5️⃣ Access the API Documentation
+- Open **Swagger UI**: [http://localhost:8000/docs](http://localhost:8000/docs)
+- Open **ReDoc**: [http://localhost:8000/redoc](http://localhost:8000/redoc)
+
+---
+
+## 📬 API Endpoints
+| Method | Endpoint          | Description              |
+|--------|------------------|--------------------------|
+| POST   | `/upload-email`  | Upload and process email |
+| GET    | `/health`        | Check API status         |
+
+---
+
+## 🛠 Environment Variables
+Create a `.env` file in the **config/** directory and add the necessary settings:
+```ini
+MODEL_PATH=/path/to/llama/model
+OCR_LANGUAGE=eng
+```
+
+---
+
+## 🔄 Updating the Project
+If you pull new changes from GitHub, remember to update dependencies:
+```sh
+ git pull origin main
+ pip install -r requirements.txt
+```
+
+---
+
+## 📤 Pushing the Project to GitHub
+After making changes, push them to GitHub:
+```sh
+ git add .
+ git commit -m "Updated project files"
+ git push origin main
+```
+
+---
+
+## 📝 Contributing
+Feel free to fork and contribute to this project! 😊
+

backend/app/api/endpoints.py

@@ -0,0 +1,82 @@
+
+from fastapi import APIRouter, HTTPException, UploadFile, File, Depends
+from typing import List, Optional
+from pydantic import BaseModel
+from datetime import datetime
+import os
+import email
+from email import policy
+from email.parser import BytesParser
+import dateutil.parser
+import io
+from app.services.classify_prompt import classify_email_with_prompt
+from app.services.email_reader import parse_email_bytes, read_emails_from_directory, parse_email
+from app.services.duplicate_checker import check_duplicate
+from app.models.email_model import EmailData
+from app.models.request_type_model import RequestTypeModel
+from app.services.gemeni_classification import analyze_intent, classify_email_gemeni, extract_text_from_attachment, get_primary_intent
+from app.services.retrieve_email_process import process_single_email
+from config import settings
+
+router = APIRouter()
+
+
+@router.post("/process-emails-upload/", response_model=List[EmailData])
+async def process_email_files(files: List[UploadFile] = File(...)):
+    """Processes multiple email files uploaded from form data."""
+    results = []
+    for file in files:
+        try:
+            file_content = await file.read()
+            email_result = await process_single_email(file_content, file.filename)
+            if email_result:
+                email_resp = EmailData(
+                    sender=email_result["sender"],
+                    subject=email_result["subject"],
+                    request_type=email_result["request_type"],
+                    sub_request_type=email_result["sub_request_type"],
+                    confidence_score=email_result["confidence_score"],
+                    duplicate_flag=email_result["duplicate_flag"],
+                )
+                results.append(email_resp)
+        except Exception as e:
+            print(f"Error processing {file.filename}: {e}")
+    return results
+
+
+@router.post("/process-email-directory/", response_model=List[EmailData])
+async def process_email_directory():
+    """Processes email files from a directory specified in an environment variable."""
+    directory_path = settings.settings.directory_path
+
+    if not directory_path:
+        raise HTTPException(status_code=400, detail="EMAIL_DIRECTORY_PATH environment variable not set.")
+
+    if not os.path.exists(directory_path) or not os.path.isdir(directory_path):
+        raise HTTPException(status_code=400, detail="Invalid directory path.")
+
+    results = []
+    email_files = []
+    for file in os.listdir(directory_path):
+        if file.endswith(".eml") or file.endswith(".msg") or file.endswith(".txt"):
+            email_files.append(os.path.join(directory_path, file))
+
+    for file_path in email_files:
+        try:
+            with open(file_path, "rb") as f:
+                file_content = f.read()
+            filename = os.path.basename(file_path)
+            email_result = await process_single_email(file_content, filename)
+            if email_result:
+                email_resp = EmailData(
+                    sender=email_result["sender"],
+                    subject=email_result["subject"],
+                    request_type=email_result["request_type"],
+                    sub_request_type=email_result["sub_request_type"],
+                    confidence_score=email_result["confidence_score"],
+                    duplicate_flag=email_result["duplicate_flag"],
+                )
+                results.append(email_resp)
+        except Exception as e:
+            print(f"Error processing {file_path}: {e}")
+    return results

backend/app/models/email_model.py

@@ -0,0 +1,10 @@
+from pydantic import BaseModel
+from typing import List, Optional
+
+class EmailData(BaseModel):
+    sender: str
+    subject: str
+    request_type: Optional[str] = None
+    sub_request_type: Optional[str] = None
+    confidence_score: Optional[float] = None
+    duplicate_flag: bool = False

backend/app/models/request_type_model.py

@@ -0,0 +1,99 @@
+class RequestTypeModel:
+    requests_datasets = [
+    {
+        "Request Type": "Adjustment",
+        "Sub-Request Type": ["N/A"],
+    },
+    {
+        "Request Type": "AU Transfer",
+        "Sub-Request Type": ["N/A"],
+    },
+    {
+        "Request Type": "Closing Notice",
+        "Sub-Request Type": ["Reallocation Fees", "Amendment Fees", "Reallocation Principal"],
+    },
+    {
+        "Request Type": "Commitment Change",
+        "Sub-Request Type": ["Cashless Roll", "Decrease", "Increase"],
+    },
+    {
+        "Request Type": "Fee Payment",
+        "Sub-Request Type": ["Ongoing Fee", "Letter of Credit Fee"],
+    },
+    {
+        "Request Type": "Money Movement - Inbound",
+        "Sub-Request Type": ["Principal", "Interest", "Principal + Interest", "Principal + Interest + Fee"],
+    },
+    {
+        "Request Type": "Money Movement - Outbound",
+        "Sub-Request Type": ["Timebound", "Foreign Currency"],
+    },
+    {
+        "Request Type": "Account Opening",
+        "Sub-Request Type": ["Checking Account", "Savings Account", "Money Market Account", "Certificate of Deposit (CD)"],
+    },
+    {
+        "Request Type": "Account Closing",
+        "Sub-Request Type": ["Checking Account", "Savings Account", "Money Market Account", "Certificate of Deposit (CD)"],
+    },
+    {
+        "Request Type": "Balance Inquiry",
+        "Sub-Request Type": ["Checking Account", "Savings Account", "Loan Account", "Credit Card Account"],
+    },
+    {
+        "Request Type": "Statement Request",
+        "Sub-Request Type": ["Checking Account", "Savings Account", "Loan Account", "Credit Card Account"],
+    },
+    {
+        "Request Type": "Transaction History Request",
+        "Sub-Request Type": ["Checking Account", "Savings Account", "Loan Account", "Credit Card Account"],
+    },
+    {
+        "Request Type": "Funds Transfer",
+        "Sub-Request Type": ["Internal Transfer", "External Transfer (ACH, Wire)"],
+    },
+    {
+        "Request Type": "Stop Payment",
+        "Sub-Request Type": ["Check", "Electronic Transfer"],
+    },
+    {
+        "Request Type": "Credit Card Application",
+        "Sub-Request Type": ["Personal", "Business"],
+    },
+    {
+        "Request Type": "Credit Limit Change",
+        "Sub-Request Type": ["Increase", "Decrease"],
+    },
+    {
+        "Request Type": "Loan Application",
+        "Sub-Request Type": ["Mortgage", "Auto Loan", "Personal Loan", "Business Loan"],
+    },
+    {
+        "Request Type": "Loan Disbursement",
+        "Sub-Request Type": ["Initial Disbursement", "Subsequent Disbursement"],
+    },
+    {
+        "Request Type": "Loan Payoff",
+        "Sub-Request Type": ["Principal", "Interest", "Fees"],
+    },
+    {
+        "Request Type": "Customer Information Update",
+        "Sub-Request Type": ["Address Change", "Phone Number Change", "Email Address Change"],
+    },
+    {
+        "Request Type": "Online/Mobile Banking Access",
+        "Sub-Request Type": ["Enrollment", "Password Reset", "Access Removal"],
+    },
+    {
+        "Request Type": "Security Request",
+        "Sub-Request Type": ["Change PIN", "Report Lost/Stolen Card", "Fraud Alert"],
+    }
+]
+    
+    @classmethod
+    def get_sub_types(cls, request_type):
+        """Retrieve sub-request types based on the request type"""
+        for item in cls.requests_datasets:
+            if item["Request Type"] == request_type:
+                return item["Sub-Request Type"]
+        return []

backend/app/services/classify_prompt.py

@@ -0,0 +1,184 @@
+import requests
+import json
+from app.models.request_type_model import RequestTypeModel
+from config import settings
+
+MODEL_NAME = settings.settings.MODEL_NAME  # e.g., "tiiuae/falcon-7b-instruct"
+HF_TOKEN = settings.settings.HUGGINGFACE_API_TOKEN  # Hugging Face token
+
+if not HF_TOKEN or HF_TOKEN == "YOUR_HUGGINGFACE_API_TOKEN":
+    print("Error: Hugging Face API token is missing. Please ensure you have a valid config.")
+    exit()
+
+API_URL = f"https://router.huggingface.co/hf-inference/models/{MODEL_NAME}/v1/chat/completions"
+HEADERS = {"Authorization": f"Bearer {HF_TOKEN}"}
+
+PROMPT_OBJECTIVE_CLASSIFICATION_RULES = """
+### Task: Email Classification 
+
+#### **Objective:** 
+Analyze the given email and classify it into the most appropriate **Request Type** and **Sub Request Type** based on its primary intent.
+
+#### **Instructions:**
+- Identify the key intent of the email.
+- Match it with one of the **Request Types** from the predefined categories.
+- Select the most relevant **Sub Request Type** for the classification.
+- If no exact match is found, choose the closest category.
+
+Only return a JSON object with the classification results.
+"""
+PROMPT_CATEGORIES = f"""
+#### **Classification Categories & Definitions:** 
+{json.dumps(RequestTypeModel.requests_datasets, indent=2)}
+"""
+
+PROMPT_OUTPUT_FORMAT = """
+#### **Output Format:** 
+Return the classification result in **pure JSON format** (without extra text or markdown).
+
+Example Output:
+{
+  "request_type": "Commitment Change",
+  "sub_request_type": "Increase",
+  "confidence_score": 0.95,
+  "email_subject": "Request for Credit Line Increase"
+}
+"""
+PROMPT_TEMPLATE = """ 
+### Task: Email Classification  
+#### **Objective:**  
+Analyze the given email and classify it into the most appropriate **Request Type** and **Sub Request Type** based on its primary intent. Ensure the response is strictly in JSON format with the specified fields.  
+
+#### **Classification Categories:**  
+Each email must be categorized under one of the following **Request Types** and corresponding **Sub Request Types**:  
+
+| Request Type              | Sub Request Type                                      |  
+|---------------------------|------------------------------------------------------|  
+| Adjustment                | N/A                                                  |  
+| AU Transfer               | N/A                                                  |  
+| Closing Notice            | Reallocation Fees, Amendment Fees, Reallocation Principal |  
+| Commitment Change         | Cashless Roll, Decrease, Increase                    |  
+| Fee Payment               | Ongoing Fee, Letter of Credit Fee                    |  
+| Money Movement - Inbound  | Principal, Interest, Principal + Interest, Principal + Interest + Fee |  
+| Money Movement - Outbound | Timebound, Foreign Currency                          |  
+
+#### **Output Format:**  
+Return the classification result strictly in **JSON format** with the following fields:  
+```json
+{
+  "request_type": "Request Type",
+  "sub_request_type": "Sub Request Type",
+  "confidence_score": Confidence Score (between 0 and 1),
+  "email_subject": "Email Subject"
+}
+### **🔹 Email for Classification:**  
+```email
+{{
+QQA Bank, N.A.
+Loan Agency Services
+
+
+ Date: 05-Feb-2025
+    TO: ABC BANK, NATIONAL ASSOCIATION
+ATTN: RAMAKRISHNA KUNCHALA
+    Fax: 877-606-9426
+      Re: ABB MID-ATLANTIC LLC $171.3MM 11-4-2022, TERM LOAN A-2
+
+Description: Facility Lender Share Adjustment
+
+BORROWER: ABB MID-ATLANTIC LIC
+DEAL NAME: ABB MID-ATLANTIC LIC $171. 3MM 11-4-2022
+
+Effective 04-Feb-2025, the Lender Shares of facility TERM LOAN A-2 have been adjusted.
+Your share of the commitment was USD 5,518,249.19. It has been Increased to USD 5,542,963.55.
+
+For: ABC BANK, NA
+
+Reference: ABIB MID-ATLANTIC LIC $171.3MM 11-4-2022,
+
+If you have any questions, please call the undersigned.
+********************************************COMMENT***************************************
+PLEASE FUND YOUR SHARE OF $24,714.36
+
+
+Bank Name: QQA Bank NA
+ABA   #   011500120
+Account #: 0026693011
+Account Name: LIQ CLO Operating Account
+Ref: ABTB Mid-Atlantic LLC
+
+********************************************************************************************
+Regards,
+
+SCOTT WALLACE
+Telephone #:
+Fax #:
+
+QQA Commercial Banking is a brand name of QQA Bank, N.A. Member FDIC
+
+}}
+
+"""
+
+def extract_json_from_response(response_text):
+    """Extract JSON response from model output."""
+    try:
+        json_start = response_text.find('{')
+        json_end = response_text.rfind('}') + 1
+        json_string = response_text[json_start:json_end]
+        return json.loads(json_string)
+    except (ValueError, json.JSONDecodeError):
+        return {"error": "Could not extract JSON from model output"}
+
+def send_to_huggingface_api(prompt):
+    """Send the prompt to Hugging Face API and get the response."""
+    try:
+        payload = {
+            "messages": [
+                {"role": "system", "content": PROMPT_OBJECTIVE_CLASSIFICATION_RULES + PROMPT_CATEGORIES + PROMPT_OUTPUT_FORMAT},
+                {"role": "user", "content": prompt}
+            ],
+            "max_tokens": 700,
+            "temperature": 0.2,
+            "top_p": 0.8,
+            "model": MODEL_NAME
+        }
+        response = requests.post(API_URL, headers=HEADERS, json=payload)
+        response.raise_for_status()
+        result = response.json()
+        
+        if "choices" in result and result["choices"]:
+            return result["choices"][0]["message"]["content"]
+        return {"error": "Unexpected API response format"}
+    
+    except requests.exceptions.RequestException as e:
+        return {"error": f"API request failed: {e}"}
+
+
+def classify_email_with_prompt(email_text):
+
+
+    final_response = send_to_huggingface_api(email_text)
+    
+    try:
+        classification = extract_json_from_response(final_response)
+        request_type = classification.get("request_type", "").strip()
+        if request_type:
+            valid_sub_types = RequestTypeModel.get_sub_types(request_type)
+            sub_request_type = classification.get("sub_request_type", "").strip()
+            if sub_request_type not in valid_sub_types:
+                classification["sub_request_type"] = valid_sub_types[0]
+        return classification
+    except Exception as e:
+        return {"error": str(e)}
+
+# Example Usage
+if __name__ == "__main__":
+    email_text = """
+    QQA Bank, N.A.
+    Loan Agency Services
+    Date: 05-Feb-2025
+    Description: Facility Lender Share Adjustment
+    """
+    classification = classify_email_with_prompt(email_text)
+    print(classification)

backend/app/services/duplicate_checker.py

@@ -0,0 +1,20 @@
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+vectorizer = TfidfVectorizer()
+email_cache = []
+
+def check_duplicate(email_text: str):
+    global email_cache
+    email_cache.append(email_text)
+
+    if len(email_cache) > 1: #check if there is more than 1 email in cache.
+        tfidf_matrix = vectorizer.fit_transform(email_cache)
+        similarity_matrix = cosine_similarity(tfidf_matrix[-1:], tfidf_matrix[:-1])
+
+        if len(similarity_matrix[0]) > 0 and max(similarity_matrix[0]) > 0.9:
+            return True, f"Similar email found with similarity {max(similarity_matrix[0])}"
+        else:
+            return False, None
+    else: #if only 1 email, then it is not a duplicate.
+        return False, None

backend/app/services/email_parser.py

@@ -0,0 +1,12 @@
+from .email_reader import read_email
+from ..models.email_model import EmailData
+
+def parse_email(file_path: str) -> EmailData:
+    email_data = read_email(file_path)
+    return EmailData(
+        sender=email_data["sender"],
+        recipient=email_data["recipient"],
+        subject=email_data["subject"],
+        body=email_data["body"],
+        attachments=email_data["attachments"]
+    )

backend/app/services/email_reader.py

@@ -0,0 +1,184 @@
+import os
+import email
+from email import policy
+from email.parser import BytesParser
+import re
+from typing import Dict, List, Optional
+import dateutil
+from fastapi import HTTPException
+
+def read_emails_from_directory(directory: str) -> List[str]:
+    """Reads email files from a given directory"""
+    emails = []
+    for file in os.listdir(directory):
+        if file.endswith(".eml") or file.endswith(".msg") or file.endswith(".txt"):
+            emails.append(os.path.join(directory, file))
+    return emails
+
+def parse_email(file_path: str) -> dict:
+    """Parses email file and extracts metadata, body, and attachments."""
+    with open(file_path, "rb") as f:
+        msg = BytesParser(policy=policy.default).parse(f)
+
+    body = ""
+    attachments = []
+    for part in msg.walk():
+        if part.get_content_type() == "text/plain":
+            body += part.get_payload(decode=True).decode("utf-8", errors="ignore")
+        elif part.get_filename():
+            attachments.append(part.get_filename())
+
+    return {
+        "sender": msg["From"],
+        "subject": msg["Subject"],
+        "date": msg["Date"],
+        "body": body,
+        "attachments": attachments
+    }
+
+def parse_email_bytes(file_content: bytes, filename: str) -> Optional[Dict]:
+    """Parses email bytes, extracts attachments, and handles email chains."""
+    try:
+        msg = email.message_from_bytes(file_content)
+        sender = msg["from"]
+        subject = msg["subject"]
+        body = ""
+        attachments = []
+        email_chain_text = ""
+
+        if msg.is_multipart():
+            for part in msg.walk():
+                content_type = part.get_content_type()
+                content_disposition = str(part.get("Content-Disposition"))
+
+                if content_type == "text/plain" and "attachment" not in content_disposition:
+                    body += part.get_payload(decode=True).decode()
+                elif content_type == "text/html" and "attachment" not in content_disposition:
+                    html = part.get_payload(decode=True).decode()
+                    body += ''.join(c if ord(c) < 128 else ' ' for c in html.replace("<br>", "\n").replace("<p>", "\n").replace("</p>","\n").replace("<div>","\n").replace("</div>","\n").replace("<span>"," ").replace("</span>", " "))
+
+                elif "attachment" in content_disposition:
+                    attachment_data = part.get_payload(decode=True)
+                    attachments.append({
+                        "filename": part.get_filename(),
+                        "content": attachment_data,
+                    })
+            email_chain_text = process_email_chain(msg) #only process if it is multipart.
+        else:
+            body = msg.get_payload(decode=True).decode()
+
+        # Handle email chains (add logic based on your needs)
+        # email_chain_text = process_email_chain(msg) #see full code in previous response.
+
+        return {
+            "sender": sender if sender else "Unknown Sender", #added default value.
+            "subject": subject if subject else "No Subject", #added default value.
+            "body": body,
+            "attachments": attachments,
+            "email_chain_text": email_chain_text
+        }
+
+    except Exception as e:
+        print(f"Error parsing email: {e}")
+        return None
+    
+
+def process_email_chain(email_message):
+    """
+    Detects and processes email chains, extracting text from each email.
+    """
+    try:
+        if isinstance(email_message, str):
+            msg = email.message_from_string(email_message)
+        else:
+            msg = email_message
+
+        chain = []
+        full_text = ""
+
+        # Check for nested email headers (From:, Date:) or quoted text
+        if has_nested_emails(msg):
+            # Parse the email and extract the chain
+            chain = extract_email_chain(msg)
+
+            for email_part in chain:
+                full_text += extract_text_from_email(email_part) + "\n"
+        else:
+            # No nested emails, just extract the text from the current email
+            full_text = extract_text_from_email(msg)
+
+        return full_text
+
+    except Exception as e:
+        print(f"Error processing email chain: {e}")
+        return extract_text_from_email(email_message) #default to just the email.
+
+def has_nested_emails(msg):
+    """
+    Detects if an email contains nested emails based on headers or quoted text.
+    """
+    body = get_email_body_text(msg)
+
+    # Check for multiple 'From:' and 'Date:' headers in the body
+    if body:
+        if len(re.findall(r"^From:.*", body, re.MULTILINE)) > 1 or \
+           len(re.findall(r"^Date:.*", body, re.MULTILINE)) > 1 or \
+           len(re.findall(r"^>.*", body, re.MULTILINE)) > 5: #arbitrary number of quoted lines.
+            return True
+    return False
+
+def extract_email_chain(msg):
+    """
+    Extracts the individual emails from a nested email chain.
+    """
+    chain = []
+    #this is a very basic attempt at parsing the email chain. It is not perfect, and will need to be improved based on specific email formatting.
+    body = get_email_body_text(msg)
+    if not body:
+        return [msg] #if no body, then return the message.
+
+    #basic email chain splitting.
+    emails = re.split(r"(^From:.*?\n^Date:.*?(\n\n|\r\n\r\n))", body, flags=re.MULTILINE | re.DOTALL)
+    if len(emails) > 1:
+        for i in range(1, len(emails), 2):
+            email_part = emails[i] + emails[i+1]
+            try:
+                chain.append(email.message_from_string(email_part))
+            except Exception as e:
+                print(f"Error parsing email part: {e}")
+                pass #if error, skip the email part.
+
+    if not chain:
+        chain = [msg] #if no chain, then return the original email.
+
+    return chain
+
+def extract_text_from_email(email_message):
+    try:
+        text = ""
+        for part in email_message.walk():
+            if part.get_content_type() == "text/plain":
+                text += part.get_payload(decode=True).decode()
+            elif part.get_content_type() == "text/html":
+                html = part.get_payload(decode=True).decode()
+                text += ''.join(c if ord(c) < 128 else ' ' for c in html.replace("<br>", "\n").replace("<p>", "\n").replace("</p>","\n").replace("<div>","\n").replace("</div>","\n").replace("<span>"," ").replace("</span>", " "))
+
+        return text
+    except Exception as e:
+        print(f"Error parsing email: {e}")
+        return ""
+    
+def get_email_body_text(msg):
+    """
+    Gets the email body text.
+    """
+    body = ""
+    if msg.is_multipart():
+        for part in msg.walk():
+            if part.get_content_type() == "text/plain":
+                body += part.get_payload(decode=True).decode()
+            elif part.get_content_type() == "text/html":
+                body += part.get_payload(decode=True).decode()
+    else:
+        body = msg.get_payload(decode=True).decode()
+    return body

backend/app/services/gemeni_classification.py

@@ -0,0 +1,160 @@
+import google.generativeai as genai
+import json
+import email
+import io
+import PyPDF2
+import docx
+import mimetypes
+   
+
+genai.configure(api_key="AIzaSyBdZsC0DG4OYjitl-OmFHLVqYOrBSFkV-c")
+
+# model = genai.GenerativeModel('gemini-pro')
+model = genai.GenerativeModel("gemini-2.0-flash-lite")
+
+classification_categories = [
+    {"request_type": "Adjustment", "sub_request_types": ["N/A"]},
+    {"request_type": "AU Transfer", "sub_request_types": ["N/A"]},
+    {"request_type": "Closing Notice", "sub_request_types": ["Reallocation Fees", "Amendment Fees", "Reallocation Principal"]},
+    {"request_type": "Commitment Change", "sub_request_types": ["Cashless Roll", "Decrease", "Increase"]},
+    {"request_type": "Fee Payment", "sub_request_types": ["Ongoing Fee", "Letter of Credit Fee"]},
+    {"request_type": "Money Movement - Inbound", "sub_request_types": ["Principal", "Interest", "Principal + Interest", "Principal + Interest + Fee"]},
+    {"request_type": "Money Movement - Outbound", "sub_request_types": ["Timebound", "Foreign Currency"]},
+    {"request_type": "Account Opening", "sub_request_types": ["Checking Account", "Savings Account", "Money Market Account", "Certificate of Deposit (CD)"]},
+    {"request_type": "Account Closing", "sub_request_types": ["Checking Account", "Savings Account", "Money Market Account", "Certificate of Deposit (CD)"]},
+    {"request_type": "Balance Inquiry", "sub_request_types": ["Checking Account", "Savings Account", "Loan Account", "Credit Card Account"]},
+    {"request_type": "Statement Request", "sub_request_types": ["Checking Account", "Savings Account", "Loan Account", "Credit Card Account"]},
+    {"request_type": "Transaction History Request", "sub_request_types": ["Checking Account", "Savings Account", "Loan Account", "Credit Card Account"]},
+    {"request_type": "Funds Transfer", "sub_request_types": ["Internal Transfer", "External Transfer (ACH, Wire)"]},
+    {"request_type": "Stop Payment", "sub_request_types": ["Check", "Electronic Transfer"]},
+    {"request_type": "Credit Card Application", "sub_request_types": ["Personal", "Business"]},
+    {"request_type": "Credit Limit Change", "sub_request_types": ["Increase", "Decrease"]},
+    {"request_type": "Loan Application", "sub_request_types": ["Mortgage", "Auto Loan", "Personal Loan", "Business Loan"]},
+    {"request_type": "Loan Disbursement", "sub_request_types": ["Initial Disbursement", "Subsequent Disbursement"]},
+    {"request_type": "Loan Payoff", "sub_request_types": ["Principal", "Interest", "Fees"]},
+    {"request_type": "Customer Information Update", "sub_request_types": ["Address Change", "Phone Number Change", "Email Address Change"]},
+    {"request_type": "Online/Mobile Banking Access", "sub_request_types": ["Enrollment", "Password Reset", "Access Removal"]},
+    {"request_type": "Security Request", "sub_request_types": ["Change PIN", "Report Lost/Stolen Card", "Fraud Alert"]}
+]
+
+
+def extract_text_from_attachment(attachment_bytes, filename):
+    file_type, _ = mimetypes.guess_type(filename)
+
+    if file_type == 'application/pdf':
+        try:
+            pdf_file = io.BytesIO(attachment_bytes)
+            pdf_reader = PyPDF2.PdfReader(pdf_file)
+            text = ""
+            for page in pdf_reader.pages:
+                text += page.extract_text() or ""
+            return text
+        except Exception as e:
+            print(f"Error extracting PDF: {e}")
+            return ""
+    elif file_type == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
+        try:
+            doc = docx.Document(io.BytesIO(attachment_bytes))
+            text = ""
+            for paragraph in doc.paragraphs:
+                text += paragraph.text + "\n"
+            return text
+        except Exception as e:
+            print(f"Error extracting Word: {e}")
+            return ""
+    else:
+        try:
+            return attachment_bytes.decode('utf-8')
+        except UnicodeDecodeError:
+            print("Unsupported attachment type or encoding.")
+            return ""
+        
+
+def extract_text_from_email(email_string):
+    try:
+        msg = email.message_from_string(email_string)
+        text = ""
+        for part in msg.walk():
+            if part.get_content_type() == "text/plain":
+                text += part.get_payload(decode=True).decode()
+            elif part.get_content_type() == "text/html":
+                #basic html removal, for more robust html, use BeautifulSoup.
+                html = part.get_payload(decode=True).decode()
+                text += ''.join(c if ord(c) < 128 else ' ' for c in html.replace("<br>", "\n").replace("<p>", "\n").replace("</p>","\n").replace("<div>","\n").replace("</div>","\n").replace("<span>"," ").replace("</span>", " "))
+
+        return text
+    except Exception as e:
+        print(f"Error parsing email: {e}")
+        return ""
+
+def analyze_intent(text):
+    prompt = f"Analyze the following text: {text}. What is the primary intent?"
+    try:
+        response = model.generate_content(prompt)
+        return response.text
+    except Exception as e:
+        print(f"Gemini API error (Intent): {e}")
+        return ""
+
+
+def classify_email_gemeni(subject, body):
+    """Classifies an email based on request type and sub-request type."""
+    results = []
+    categories_string = str(classification_categories)
+    PROMPT = f"""
+    Analyze the following email and classify it into the most appropriate Request Type and Sub Request Type based on its primary intent.
+
+    Classification Categories:
+    {categories_string}
+
+    Email Subject: {subject}
+    Email Content: {body}
+
+    Output Format:
+    {{
+        "request_type": "Request Type",
+        "sub_request_type": "Sub Request Type",
+        "confidence_score": "Confidence Score (between 0 and 1)",
+    }}
+    """
+   
+
+    
+    response = model.generate_content(PROMPT)
+
+    if response and hasattr(response, "_result"):
+        text_response = response._result.candidates[0].content.parts[0].text
+        text_response = text_response.strip().replace("```json", "").replace("```", "").strip()
+        try:
+            parsed_json = json.loads(text_response)
+
+            request_type = parsed_json.get("request_type", "Unknown")
+            sub_request_type = parsed_json.get("sub_request_type", "Unknown")
+            confidence_score = parsed_json.get("confidence_score", "Unknown")
+
+            results.extend([request_type,sub_request_type,confidence_score])
+            return results
+
+        except json.JSONDecodeError as e:
+            print(f"Exception during JSON Parsing: {e}")
+    else:
+        print("We did not get response.")
+
+
+
+
+def get_primary_intent(email_content, attach_content):
+    """Detects primary intent when multiple requests are present."""
+
+    model = genai.GenerativeModel("gemini-2.0-pro-exp-02-05")
+
+    prompt = f"""
+    Analyze the following email and document, and determine which one has the primary intent. 
+    email::{email_content} 
+    document: {attach_content}
+    """
+    response = model.generate_content(prompt)
+    return response.text  # Extracted primary intent
+
+
+

backend/app/services/ocr_processor.py

@@ -0,0 +1,16 @@
+import pytesseract
+import pdfplumber
+from PIL import Image
+
+def extract_text_from_pdf(pdf_path: str) -> str:
+    """Extracts text from a PDF file."""
+    text = ""
+    with pdfplumber.open(pdf_path) as pdf:
+        for page in pdf.pages:
+            text += page.extract_text() + "\n"
+    return text.strip()
+
+def extract_text_from_image(image_path: str) -> str:
+    """Extracts text from an image file using OCR."""
+    image = Image.open(image_path)
+    return pytesseract.image_to_string(image)

backend/app/services/retrieve_email_process.py

@@ -0,0 +1,52 @@
+
+from typing import Optional
+from app.services.duplicate_checker import check_duplicate
+from app.services.email_reader import parse_email_bytes
+from app.services.gemeni_classification import classify_email_gemeni, extract_text_from_attachment
+
+
+async def process_single_email(file_content: bytes, filename: str) -> Optional[dict]:
+    """Processes a single email content."""
+    email_data = parse_email_bytes(file_content, filename)
+    if email_data:
+        attachment_text = ""
+        for attachment in email_data["attachments"]:
+            attachment_text += extract_text_from_attachment(attachment["content"], attachment["filename"])
+
+        email_chain_text = email_data["email_chain_text"]
+        email_body_text = email_data["body"]
+
+        # 1. Separate Classification:
+        document_result = classify_email_gemeni(email_data["subject"], attachment_text) if attachment_text else ("Unknown", "Unknown", "0")
+        email_chain_result = classify_email_gemeni(email_data["subject"], email_chain_text) if email_chain_text else ("Unknown", "Unknown", "0")
+        primary_email_result = classify_email_gemeni(email_data["subject"], email_body_text)
+
+        # 2. Confidence Score Comparison:
+        document_confidence = float(document_result[2])
+        email_chain_confidence = float(email_chain_result[2])
+        primary_email_confidence = float(primary_email_result[2])
+
+        best_result = primary_email_result  # Default to email body
+        if document_confidence > primary_email_confidence and document_confidence > email_chain_confidence:
+            best_result = document_result
+        elif email_chain_confidence > primary_email_confidence and email_chain_confidence > document_confidence:
+            best_result = email_chain_result
+
+        # 3. Refined Classification:
+        request_type = best_result[0]
+        sub_request_type = best_result[1]
+        confidence_score = best_result[2]
+
+        duplicate_flag, duplicate_reason = check_duplicate(email_data["body"])
+        email_obj = {
+            "sender": email_data["sender"],
+            "subject": email_data["subject"],
+            "request_type": request_type,
+            "sub_request_type": sub_request_type,
+            "confidence_score": confidence_score,
+            "duplicate_flag": duplicate_flag,
+        }
+        return email_obj
+    else:
+        print(f"Parsing failed for file: {filename}")
+        return None

backend/config/settings.py

@@ -0,0 +1,34 @@
+import os
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+
+class Settings:
+    ENV = os.getenv("ENV", "local")  # Default to "local" if ENV is not set
+    # API Token
+    HUGGINGFACE_API_TOKEN = os.getenv("HUGGINGFACE_API_TOKEN") or os.environ.get("HUGGINGFACE_API_TOKEN")
+
+    # Model Path
+    MODEL_NAME = os.getenv("MODEL_NAME") or os.environ.get("MODEL_NAME") # Default if not set
+    OCR_LANGUAGE = os.getenv("OCR_LANGUAGE", "eng")
+    directory_path = os.getenv("EMAIL_DIRECTORY_PATH") or os.environ.get("EMAIL_DIRECTORY_PATH")  # Make configurable
+    #MODEL_NAME = "meta-llama/Llama-2-7b"    
+
+     # Ensure model path exists
+    # if not os.path.exists(MODEL_PATH):
+    #     os.makedirs(MODEL_PATH, exist_ok=True)
+
+    # # Validate required variables
+    # if not HUGGINGFACE_API_TOKEN:
+    #     raise ValueError("Missing HUGGINGFACE_API_TOKEN. Please set it in the environment variables or .env file.")
+
+    # Debugging info
+    print(f"Running in {ENV} mode with model path: {MODEL_NAME}")
+    print(f"Running in {ENV} mode with TOKEN: {HUGGINGFACE_API_TOKEN}")
+
+settings = Settings()
+
+
+

backend/main.py

@@ -0,0 +1,24 @@
+from fastapi import APIRouter, FastAPI
+from app.api.endpoints import router 
+from fastapi.middleware.cors import CORSMiddleware
+
+
+app = FastAPI(title="Gen AI Email Processing API")  # This creates the FastAPI app instance
+
+#router = APIRouter()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Replace "*" with your frontend domain in production
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Include API routes from endpoints.py
+app.include_router(router, prefix="/api")  # You can remove prefix if not needed
+
+@app.get("/")
+async def root():
+    return {"message": "Hello from FastAPI"}
+
+#app.include_router(router)

backend/requirements.txt

@@ -0,0 +1,16 @@
+
+fastapi==0.115.11
+uvicorn==0.34.0
+pydantic==2.10.6
+pdfplumber==0.11.5
+python-docx==1.1.2
+pytesseract==0.3.13
+numpy==2.2.4
+pandas==2.2.3
+scikit-learn==1.6.1
+tika==2.6.0
+unstructured==0.17.2
+pypdf==5.4.0
+langchain==0.3.21
+python-dotenv
+PyPDF2