nit comment

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/*.JPG filter=lfs diff=lfs merge=lfs -text
+examples/*.jpg filter=lfs diff=lfs merge=lfs -text
+examples/*.* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,50 @@
+# These are some examples of commonly ignored file patterns.
+# You should customize this list as applicable to your project.
+# Learn more about .gitignore:
+#     https://www.atlassian.com/git/tutorials/saving-changes/gitignore
+
+# Node artifact files
+node_modules/
+dist/
+
+# Compiled Java class files
+*.class
+
+# Compiled Python bytecode
+*.py[cod]
+
+# Log files
+*.log
+
+# Package files
+*.jar
+
+# Maven
+target/
+dist/
+
+# JetBrains IDE
+.idea/
+
+# Unit test reports
+TEST*.xml
+
+# Generated by MacOS
+.DS_Store
+
+# Generated by Windows
+Thumbs.db
+
+# Applications
+*.app
+*.exe
+*.war
+
+# Large media files
+*.mp4
+*.tiff
+*.avi
+*.flv
+*.mov
+*.wmv
+

README.md

@@ -1,12 +1,13 @@
 ---
-title: ReceiptSplitAI
-emoji: 🌍
-colorFrom: green
-colorTo: indigo
+title: ReceiptAI
+emoji: 🏢
+colorFrom: pink
+colorTo: pink
 sdk: gradio
-sdk_version: 4.44.1
+sdk_version: 4.44.0
 app_file: app.py
 pinned: false
+license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

algorithm/product.py

@@ -0,0 +1,8 @@
+class Product:
+    def __init__(self, name, price, quantity):
+        self.name = name
+        self.price = price
+        self.quantity = quantity
+
+    def __repr__(self):
+        return f"Product(name={self.name}, price={self.price}, quantity={self.quantity})"

algorithm/receipt.py

@@ -0,0 +1,8 @@
+class Receipt:
+    def __init__(self, products, tax, total_amount):
+        self.products = products
+        self.tax = tax
+        self.total_amount = total_amount
+
+    def __repr__(self):
+        return f"Receipt(products={self.products}, tax={self.tax}, total_amount={self.total_amount})"

algorithm/receipt_calculation.py

@@ -0,0 +1,128 @@
+import math
+import copy
+
+from algorithm.product import Product
+
+
+def calculate_dish_price_with_taxes(_products, taxes, grand_total):
+    payment_total = round(grand_total - taxes, 2)
+    grand_total = round(payment_total, 2) + round(taxes, 2)
+    _product_with_taxes = copy.deepcopy(_products)
+    for _product in _product_with_taxes:
+        _product.price = round(((_product.price / payment_total) * grand_total), 5)
+    return _product_with_taxes, grand_total
+
+
+def round_up_two_decimals(_products_total):
+    _product_with_taxes_rounded = copy.deepcopy(_products_total)
+    for _product in _product_with_taxes_rounded:
+        _product.price = math.ceil(_product.price * 100) / 100
+
+    return _product_with_taxes_rounded
+
+
+def first_algorithm(_products_total_rounded, receipt_subtotal):
+    current_total = 0
+    for _product in _products_total_rounded:
+        current_total += _product.price
+
+    current_total = round(current_total, 2)
+
+    difference = current_total - receipt_subtotal
+
+    corrections = copy.deepcopy(_products_total_rounded)
+
+    for _product in corrections:
+        _product.price = round((_product.price / current_total) * difference, 2)
+
+    for i in range(len(_products_total_rounded)):
+        _products_total_rounded[i].price = round(_products_total_rounded[i].price - corrections[i].price, 2)
+
+    _final_total = 0
+    for _product in _products_total_rounded:
+        _final_total += _product.price
+
+    return _products_total_rounded, _final_total
+
+
+def fractional_part_rest(value):
+    fraction_str = f"{value:.10f}".split('.')[1]
+    rest_of_digits = fraction_str[2:]
+    return float(rest_of_digits)
+
+
+def second_algorithm(_products_total, receipt_total):
+    _products_total_rounded = round_up_two_decimals(_products_total)
+
+    current_total = 0
+    for _product in _products_total_rounded:
+        current_total += _product.price
+
+    if current_total == receipt_total:
+        return _products_total_rounded, receipt_total
+
+    difference = current_total - receipt_total
+    difference = round(difference, 2)
+
+    fractional_parts = copy.deepcopy(_products_total)
+
+    for _product in fractional_parts:
+        _product.price = fractional_part_rest(_product.price) - math.ceil(_product.price)
+
+    fractional_parts = sorted(fractional_parts, key=lambda p: p.price, reverse=False)
+
+    for i in range(len(fractional_parts)):
+        if difference <= 0:
+            break
+        _products_total_rounded[i].price -= 0.01
+        difference -= 0.01
+
+    _final_total = 0
+    for _product in _products_total_rounded:
+        _final_total += _product.price
+
+    for _product in _products_total_rounded:
+        _product.price = round(_product.price, 2)
+
+    return _products_total_rounded, _final_total
+
+
+def calculate_tips_and_taxes(items_table, total_amount, tax, tips):
+    products = []
+
+    if items_table[0][0] == "No items":
+        return products, 0
+
+    if total_amount == "Not specified" or total_amount is None:
+        total_amount = "0.0"
+
+    if tax == "Not specified" or tax is None:
+        tax = "0.0"
+
+    if tips == "Not specified" or tips is None:
+        tax = "0.0"
+
+    for item in items_table:
+        price = item[5]
+        if price == "Not specified":
+            price = "0.0"
+        item_value = float(str(price).replace(",", ".")) if item[5] is not None else 0.0
+        products.append(Product(item[0], item_value, item[3]))
+
+    sum_of_product_prices = 0
+    for _product in products:
+        sum_of_product_prices += _product.price
+
+    sum_of_product_prices = round(float(sum_of_product_prices), 2)
+    total_amount = round(float(str(total_amount).replace(",", ".")), 2)
+    tips = round(float(str(tips).replace(",", ".")), 2)
+    tax = round(tips + round(float(str(tax).replace(",", ".")), 2), 2)
+    if round(float(total_amount), 2) != round(float(sum_of_product_prices) + float(tax), 2):
+        return products, sum_of_product_prices
+
+    products_total, subtotal = calculate_dish_price_with_taxes(products, taxes=float(tax),
+                                                               grand_total=float(total_amount))
+    final_prices, final_total = second_algorithm(products_total, subtotal)
+
+    final_total = round(final_total, 2)
+    return final_prices, final_total

app.py

@@ -0,0 +1,263 @@
+import json
+import pandas as pd
+import os
+from datetime import datetime
+
+import gradio as gr
+from PIL import Image
+
+from google_drive_client import GoogleDriveClient
+from openai_service import OpenAIService
+from utils import read_prompt_from_file, process_receipt_json, encode_image_to_jpeg_base64, save_to_excel, encode_image_to_webp_base64
+from vertex_ai_service import VertexAIService
+
+model_names = ["gemini-1.5-flash", "gemini-1.5-pro", "gpt-4o-mini", "gpt-4o", "gpt-4-turbo"]
+prompt_names = ["prompt_v1", "prompt_v2", "prompt_v3"]
+example_list = [["./examples/" + example] for example in os.listdir("examples")]
+prompt_default = read_prompt_from_file("prompt_v1.txt")
+system_instruction = read_prompt_from_file("system_instruction.txt")
+
+
+def process_image(input_image, model_name, prompt_name, temperatura, system_instruction=None, current_prompt_text=None):
+    # print(model_name)
+    # print(prompt_name)
+    # print(temperatura)
+    # print(custom_prompt_text)
+    if system_instruction is None:
+        system_instruction = ""
+    if input_image is None:
+        return None, "No objects detected."
+    if input_image is None:
+        return json.dumps({"error": "No prompt provided."})
+    if prompt_name is None:
+        prompt_name = "prompt_v1"
+        prompt_file = f"{prompt_name}.txt"
+        prompt = read_prompt_from_file(prompt_file)
+    if prompt_name is None:
+        current_prompt_text = prompt_default
+
+    # if prompt_name != "custom":
+    #     prompt_file = f"{prompt_name}.txt"
+    #     prompt = read_prompt_from_file(prompt_file)
+    # else:
+    #     if current_prompt_text is None or current_prompt_text.strip() == "":
+    #         return json.dumps({"error": "No prompt provided."})
+    prompt = current_prompt_text
+    # print(prompt)
+    print("file name:", input_image)
+    print("model_name:", model_name)
+    print("prompt_name:", prompt_name)
+    print("Temperatura:", temperatura)
+
+    # base64_image = encode_image_from_gradio(input_image)
+    base64_image = encode_image_to_webp_base64(input_image)
+
+    try:
+        if model_name.startswith("gpt"):
+            # result = gpt_process_image(base64_image, model_name, prompt, system_instruction, temperatura)
+            result = open_ai_client.process_image(base64_image, model_name, prompt, system_instruction, temperatura)
+        else:
+            result = vertex_ai_client.process_image(base64_image, model_name, prompt, system_instruction,
+                                                    temperatura)
+        parsed_result = json.loads(result)
+        result = json.dumps(parsed_result, ensure_ascii=False, indent=4)
+        # result = result.encode('utf-8').decode('unicode_escape')
+        print(result)
+    except Exception as e:
+        print(f"Exception occurred: {e}")
+        result = json.dumps({"error": "Error processing: Check prompt or images"})
+
+    # print (result)
+    store_info, items_table, message = process_receipt_json(result)
+    print(store_info)
+    print(items_table)
+
+    return model_name, result, store_info, items_table, message, gr.update(interactive=True),  gr.update(interactive=True),  gr.update(interactive=True)
+
+
+def save_flag_data(save_type, image, model_name, prompt_name, temperatura, current_prompt_text, model_output, json_output,
+                   store_info_output, items_list, comments_output, system_instruction, flagging_dir="custom_flagged_data"):
+    save_button_update = gr.update(interactive=False)
+    image_link, json_link, excel_link = None, None, None
+    try:
+
+        # List files in the directory
+        try:
+            files = [f for f in os.listdir(flagging_dir) if os.path.isfile(os.path.join(flagging_dir, f))]
+            if files:
+                print("Files in directory:", flagging_dir)
+                for file in files:
+                    print(file)
+            else:
+                print(f"No files found in directory: {flagging_dir}")
+        except Exception as e:
+            print(f"Error listing files in directory: {e}")
+
+        image_file_path = image
+        print("save_type:", save_type)
+        print("Image File Path:", image)
+        print("prompt_name:", prompt_name)
+        print("Model Name:", model_name)
+        print("Result as JSON:", json_output)
+        print("comments:", comments_output)
+        print("system_instruction:", system_instruction)
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        original_filename = os.path.basename(image_file_path)
+        filename, file_extension = os.path.splitext(original_filename)
+        base_filename = f"{filename}_{model_name}_{prompt_name}_{timestamp}"
+
+        # Save image
+        image_save_path = os.path.join(flagging_dir, f"{base_filename}{file_extension}")
+        image = Image.open(image_file_path)
+        image.save(image_save_path)
+
+        if os.path.exists(image_save_path):
+            saved_image = Image.open(image_save_path)
+            image_size = saved_image.size
+            print(f"Image saved at: {image_save_path}, Size: {image_size}")
+        else:
+            print(f"Failed to save image at: {image_save_path}")
+            return 0
+
+        # Save result as JSON
+        json_file_path = os.path.join(flagging_dir, f"{base_filename}.json")
+        data_to_save = {
+            "image_name": f"{base_filename}{file_extension}",
+            "prompt_name": prompt_name,
+            "system_instruction": system_instruction,
+            "prompt": current_prompt_text,
+            "model_name": model_name,
+            "result_json": json_output,
+            "comments": comments_output,
+            "save_type": save_type
+        }
+
+        data_to_save_encode = json.dumps(data_to_save, ensure_ascii=False, indent=4)
+        print("data_to_save_encode: ", data_to_save_encode)
+
+        with open(json_file_path, 'w', encoding='utf-8') as json_file:
+            json_file.write(data_to_save_encode)
+
+        excel_file_path = os.path.join(flagging_dir, f"{base_filename}.xlsx")
+        save_to_excel(json_output, excel_file_path)
+
+        # Upload files to Google Drive
+        google_drive_client_current = GoogleDriveClient(json_key_path='GOOGLE_SERVICE_ACCOUNT_KEY.json')
+        if google_drive_client_current:
+            try:
+                image_folder_id = '19yiqYX_Z1rbHDxnFLJTvji0VcbhSO9cq'
+                image_link = google_drive_client_current.upload_file(image_save_path, image_folder_id)
+                json_link = google_drive_client_current.upload_file(json_file_path, image_folder_id)
+                excel_link = google_drive_client_current.upload_file(excel_file_path, image_folder_id)
+                print(f"Image uploaded to Google Drive. Link: {image_link}")
+                print(f"JSON file uploaded to Google Drive. Link: {json_link}")
+                print(f"Excel file uploaded to Google Drive. Link: {excel_link}")
+            except Exception as e:
+                print(f"Error uploading files to Google Drive: {e}")
+        else:
+            print(f"Error google_drive_client does not available")
+
+    except Exception as e:
+        print(f"Error while saving flag data: {e}")
+    links = f"Image: {image_link}\nJSON: {json_link}\nExcel: {excel_link}"
+    return save_button_update, save_button_update, save_button_update, links
+
+def update_prompt_from_radio(prompt_name):
+    if prompt_name == "prompt_v1":
+        return read_prompt_from_file("prompt_v1.txt")
+    elif prompt_name == "prompt_v2":
+        return read_prompt_from_file("prompt_v2.txt")
+    elif prompt_name == "prompt_v3":
+        return read_prompt_from_file("prompt_v3.txt")
+    else:
+        return read_prompt_from_file("prompt_v1.txt")
+
+google_drive_client = GoogleDriveClient(json_key_path='GOOGLE_SERVICE_ACCOUNT_KEY.json')
+vertex_ai_client = VertexAIService(json_key_path='GOOGLE_VERTEX_AI_KEY.json')
+
+key = None
+key_file_path = 'OPENAI_AI_KEY.txt'
+if os.path.exists(key_file_path):
+    try:
+        with open(key_file_path, 'r') as key_file:
+            key = key_file.read().strip()
+    except Exception as e:
+        print(f"Error reading file: {e}")
+
+open_ai_client = OpenAIService(api_key=key)
+
+with gr.Blocks() as iface:
+    gr.Markdown("# ReceptAI")
+    gr.Markdown("ReceptAI")
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            image_input = gr.Image(type="filepath")
+            model_radio = gr.Radio(model_names, label="Choose model", value=model_names[0])
+            prompt_radio = gr.Radio(prompt_names, label="Choose prompt", value=prompt_names[0])
+            temperature_slider = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label="Temperatura", value=0.0)
+            system_instruction = gr.Textbox(label="System Instruction", visible=True, value=system_instruction)
+            custom_prompt = gr.Textbox(label="prompt text", visible=True, value=prompt_default)
+            with gr.Row():
+                submit_button = gr.Button("Submit")
+
+        with gr.Column(scale=2):
+            model_output = gr.Textbox(label="MODEL", lines=1, interactive=False)
+            json_output = gr.Textbox(label="Result as json")
+            store_info_output = gr.Textbox(label="Store Information", lines=4)
+            items_list = gr.Dataframe(
+                headers=["Item Name", "Category", "Unit Price", "Quantity", "Unit", "Total Price", "Discount", "Grand Total"],
+                label="Items List")
+            comments_output = gr.Textbox(label="Comments", visible=True, lines=4, interactive=True)
+            with gr.Row():
+                save_good_button = gr.Button(value="Save as Good", interactive=False)
+                save_average_button = gr.Button(value="Save as Average" , interactive=False)
+                save_poor_button = gr.Button(value="Save as Poor", interactive=False)
+            file_links_output = gr.Textbox(label="File Links", interactive=False, visible=True)
+    submit_button.click(fn=process_image,
+                        inputs=[image_input, model_radio, prompt_radio, temperature_slider, system_instruction,
+                                custom_prompt],
+                        outputs=[model_output, json_output, store_info_output, items_list, comments_output,
+                                 save_good_button, save_average_button, save_poor_button])
+    common_inputs = [image_input, model_radio, prompt_radio, temperature_slider, custom_prompt, model_output,
+                     json_output, store_info_output, items_list, comments_output, system_instruction]
+
+
+    def save_flag_data_wrapper(save_type, image, model_name, prompt_name, temperatura, custom_prompt, model_output,
+                               json_output, store_info_output, items_list, comments_output, system_instruction):
+        # Ensure that `image` is a file path and not an object.
+        image_file_path = image  # Gradio returns the path as a string
+        model_name_value = model_name  # Extract selected value
+        prompt_name_value = prompt_name  # Extract selected value
+
+        # The following variables should be passed as the values they hold
+        save_good_update, save_avg_update, save_poor_update, file_links = save_flag_data(
+            save_type, image, model_name, prompt_name, temperatura, custom_prompt, model_output, json_output,
+            store_info_output, items_list, comments_output, system_instruction
+        )
+        return save_good_update, save_avg_update, save_poor_update, file_links
+
+    # Use the same common_inputs for all buttons but ensure the correct values are passed
+    save_good_button.click(
+        fn=lambda *args: save_flag_data_wrapper("Good", *args),
+        inputs=common_inputs,
+        outputs=[save_good_button, save_average_button, save_poor_button, file_links_output]
+    )
+
+    save_average_button.click(
+        fn=lambda *args: save_flag_data_wrapper("Average", *args),
+        inputs=common_inputs,
+        outputs=[save_good_button, save_average_button, save_poor_button, file_links_output]
+    )
+
+    save_poor_button.click(
+        fn=lambda *args: save_flag_data_wrapper("Poor", *args),
+        inputs=common_inputs,
+        outputs=[save_good_button, save_average_button, save_poor_button, file_links_output]
+    )
+    prompt_radio.change(fn=update_prompt_from_radio, inputs=[prompt_radio], outputs=[custom_prompt])
+    gr.Examples(examples=example_list,
+                inputs=[image_input, model_radio, prompt_radio, temperature_slider, custom_prompt])
+
+iface.launch()

google_drive_client.py

@@ -0,0 +1,94 @@
+import os
+import tempfile
+from google.oauth2 import service_account
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload
+from googleapiclient.errors import HttpError
+
+class GoogleDriveClient:
+    def __init__(self, json_key_path=None, json_key_env_var='GOOGLE_SERVICE_ACCOUNT_KEY'):
+        """
+        Initializes the Google Drive client.
+
+        :param json_key_path: Path to the JSON file (optional)
+        :param json_key_env_var: Environment variable name containing the JSON key (default 'GOOGLE_SERVICE_ACCOUNT_KEY')
+        """
+        self.json_key_path = json_key_path or os.getenv('GOOGLE_VERTEX_KEY_PATH')
+        self.json_key_env_var = json_key_env_var
+        self.service = self._authenticate_google_drive()
+
+    def _authenticate_google_drive(self):
+        """
+        Authenticates using the JSON key from a file or environment variable.
+
+        :return: Google Drive API service object
+        """
+        if self.json_key_path and os.path.isfile(self.json_key_path):
+            # Authenticate using the file
+            creds = service_account.Credentials.from_service_account_file(
+                self.json_key_path,
+                scopes=['https://www.googleapis.com/auth/drive.file']
+            )
+        else:
+            # Get JSON key from environment variable
+            json_key = os.getenv(self.json_key_env_var)
+
+            if not json_key:
+                raise ValueError(f"Environment variable {self.json_key_env_var} is not set.")
+
+            # Save JSON key to a temporary file
+            with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+                temp_file.write(json_key.encode('utf-8'))
+                temp_file_path = temp_file.name
+
+            # Create credentials object
+            creds = service_account.Credentials.from_service_account_file(
+                temp_file_path,
+                scopes=['https://www.googleapis.com/auth/drive.file']
+            )
+
+            # Remove the temporary file after creating the credentials object
+            os.remove(temp_file_path)
+
+        # Create and return the Google Drive API service object
+        return build('drive', 'v3', credentials=creds)
+
+    def upload_file(self, file_path, folder_id):
+        """
+        Uploads a file to Google Drive.
+
+        :param file_path: Path to the local file
+        :param folder_id: ID of the folder on Google Drive
+        :return: URL of the uploaded file or error message
+        """
+        try:
+            file_metadata = {
+                'name': os.path.basename(file_path),
+                'parents': [folder_id]  # ID of the folder to upload the file to
+            }
+            media = MediaFileUpload(file_path, mimetype='application/octet-stream')
+
+            # Upload the file
+            file = self.service.files().create(body=file_metadata, media_body=media, fields='id').execute()
+
+            # Get the file ID and construct the URL
+            file_id = file.get('id')
+            file_link = f'https://drive.google.com/file/d/{file_id}/view?usp=drive_link'
+
+            return file_link
+        except HttpError as error:
+            return f"An error occurred: {error}"
+
+# Example usage
+if __name__ == '__main__':
+    # Path to the local file
+    file_path = './file.txt'
+
+    # ID of the folder on Google Drive
+    folder_id = '19yiqYX_Z1rbHDxnFLJTvji0VcbhSO9cq'
+
+    # Initialize the client and upload the file
+    client = GoogleDriveClient(json_key_path='GOOGLE_SERVICE_ACCOUNT_KEY.json')
+    file_link = client.upload_file(file_path, folder_id)
+    print(f'File upload result: {file_link}')
+

gpt_processing.py

@@ -0,0 +1,48 @@
+import json
+import os
+
+from openai import OpenAI
+
+openai_api_key = os.environ.get("OPENAI_API_KEY")
+if openai_api_key:
+    print("OPENAI_API_KEY found.")
+    # print(openai_api_key)
+else:
+    print("OPENAI_API_KEY not found.")
+
+client = OpenAI(
+    api_key=openai_api_key
+)
+
+
+def gpt_process_image(input_image64, model_name, prompt, system_instruction="", temperatura=0.0):
+    if input_image64 is None:
+        return None, "No objects detected."
+    # print (prompt)
+    print("model_name:", model_name)
+    print("Temperatura:", temperatura)
+
+    response = client.chat.completions.create(
+        model=model_name,
+        messages=[
+            {"role": "system", "content": f"{system_instruction}"},
+            {"role": "user", "content": [
+                {"type": "text", "text": f"{prompt}"},
+                {"type": "image_url", "image_url": {
+                    "url": f"data:image/jpg;base64,{input_image64}"}
+                 }
+            ]}
+        ],
+        temperature=temperatura,
+        response_format={"type": "json_object"}
+    )
+
+    json_content = json.loads(response.choices[0].message.content)
+
+    json_content['input_tokens'] = response.usage.prompt_tokens
+    json_content['output_tokens'] = response.usage.completion_tokens
+    json_content['total_tokens'] = response.usage.total_tokens
+
+    print(json_content)
+
+    return json.dumps(json_content, indent=4)

openai_service.py

@@ -0,0 +1,63 @@
+import json
+import base64
+import os
+from openai import OpenAI
+
+class OpenAIService:
+    def __init__(self, api_key=None):
+        self.api_key = api_key or os.environ.get("OPENAI_API_KEY")
+        if self.api_key:
+            print("OPENAI_API_KEY was found.")
+        else:
+            raise ValueError("OPENAI_API_KEY not found.")
+
+        self.client = OpenAI(api_key=self.api_key)
+
+    def process_image(self, input_image64, model_name, prompt, system = "You are receipt recognizer!", temperatura = 0.0):
+        if not input_image64:
+            return None, "No objects detected."
+
+        print("Model name:", model_name)
+        print("system:", system)
+        print("Temperature:", temperatura)
+
+        response = self.client.chat.completions.create(
+            model=model_name,
+            messages=[
+                {"role": "system", "content": f"{system}"},
+                {"role": "user", "content": [
+                    {"type": "text", "text": f"{prompt}"},
+                    {"type": "image_url", "image_url": {
+                        "url": f"data:image/webp;base64,{input_image64}"}
+                     }
+                ]}
+            ],
+            temperature=temperatura,
+            response_format={"type": "json_object"}
+        )
+
+        json_content = json.loads(response.choices[0].message.content)
+
+        json_content['input_tokens'] = response.usage.prompt_tokens
+        json_content['output_tokens'] = response.usage.completion_tokens
+        json_content['total_tokens'] = response.usage.total_tokens
+
+        print(json_content)
+
+        return json.dumps(json_content, indent=4)
+
+if __name__ == "__main__":
+    processor = OpenAIService()
+
+    # Image processing
+    image_path = "./examples/lidl2.jpg"
+    with open(image_path, "rb") as image_file:
+        input_image64 = base64.b64encode(image_file.read()).decode('utf-8')
+
+    system = "You are receipt recognizer "
+    prompt = "Recognize the receipt and provide result as json "
+    result = processor.process_image(input_image64, "gpt-4o-mini", prompt, system, 0.0)
+    print(f'Image processing result: {result}')
+    decoded_string = result.encode('utf-8').decode('unicode_escape')
+
+    print(result)

process_images_gemini_multithread.py

@@ -0,0 +1,100 @@
+import os
+import base64
+import json
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from utils import encode_image_to_jpeg_base64
+# Import your VertexAIService class
+from vertex_ai_service import VertexAIService  # Replace with the module name where VertexAIService is defined
+
+
+def process_image_file(file_path, client):
+    """
+    Processes an image file and returns the path to the JSON file with results.
+
+    :param file_path: Path to the image file
+    :param client: Instance of VertexAIService
+    :return: Path to the saved JSON file
+    """
+    input_image64 = encode_image_to_jpeg_base64(file_path)
+
+    # Processing settings
+    prompt = "Read the text"
+    system = "You are receipt recognizer"
+
+    # Call image processing
+    result_img, error = client.process_image(input_image64, "gemini-1.5-flash", prompt, system, 0.0)
+
+    if error:
+        raise Exception(error)
+
+    # Create path for JSON file
+    json_file_path = os.path.splitext(file_path)[0] + ".json"
+
+    # Write the result to a JSON file
+    with open(json_file_path, 'w', encoding='utf-8') as json_file:
+        json_file.write(result_img)
+
+    return json_file_path
+
+
+def process_folder_images(folder_path, json_key_path):
+    """
+    Processes all images in the specified folder concurrently.
+
+    :param folder_path: Path to the folder containing images
+    :param json_key_path: Path to the JSON key for Vertex AI authentication
+    """
+    # Initialize Vertex AI client
+    client = VertexAIService(json_key_path=json_key_path)
+
+    # Get a list of all images in the folder
+    image_files = [os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith(('.jpg', '.jpeg', '.png'))]
+
+    # Use ThreadPoolExecutor for parallel processing
+    with ThreadPoolExecutor() as executor:
+        # future_to_file = {executor.submit(process_image_file, file_path, client): file_path for file_path in image_files}
+        future_to_file = {executor.submit(process_image_file_with_retry, file_path, client, 5): file_path for file_path in image_files}
+
+
+        for future in as_completed(future_to_file):
+            file_path = future_to_file[future]
+            try:
+                json_file_path = future.result()
+                print(f"Processed: {file_path}, result saved to: {json_file_path}")
+            except Exception as exc:
+                print(f"Failed to process file {file_path}. Error: {exc}")
+
+import random
+
+def process_image_file_with_retry(file_path, client, max_retries=5):
+    """
+    Processes an image file with retry logic and returns the path to the JSON file with results.
+
+    :param file_path: Path to the image file
+    :param client: Instance of VertexAIService
+    :param max_retries: Maximum number of retries for handling quota errors
+    :return: Path to the saved JSON file
+    """
+    retries = 0
+    while retries < max_retries:
+        try:
+            json_file_path = process_image_file(file_path, client)
+            return json_file_path
+        except Exception as exc:
+            if "429" in str(exc):
+                retries += 1
+                wait_time = 2 ** retries + random.uniform(0, 1)
+                print(f"Quota exceeded for {file_path}. Retrying in {wait_time:.2f} seconds... (Attempt {retries}/{max_retries})")
+                time.sleep(wait_time)
+            else:
+                raise exc
+
+    raise Exception(f"Max retries exceeded for file {file_path}")
+
+# Call the function to process the folder
+if __name__ == '__main__':
+    folder_path = './examples'  # Set the path to your folder
+    json_key_path = 'GOOGLE_VERTEX_AI_KEY.json'  # Set the path to your JSON key
+    process_folder_images(folder_path, json_key_path)

process_images_gemini_one_hread.py

@@ -0,0 +1,65 @@
+import os
+import base64
+import json
+import time
+
+from utils import encode_image_to_jpeg_base64
+# Import your VertexAIService class
+from vertex_ai_service import VertexAIService  # Replace with the module name where VertexAIService is defined
+
+
+def process_image_file(file_path, client):
+    """
+    Processes an image file and returns the path to the JSON file with results.
+
+    :param file_path: Path to the image file
+    :param client: Instance of VertexAIService
+    :return: Path to the saved JSON file
+    """
+    input_image64 = encode_image_to_jpeg_base64(file_path)
+
+    # Processing settings
+    prompt = "Read the text"
+    system = "You are receipt recognizer"
+
+    # Call image processing
+    result_img = client.process_image(input_image64, "gemini-1.5-flash", prompt, system, 0.0)
+
+    # Create path for JSON file
+    json_file_path = os.path.splitext(file_path)[0] + ".json"
+
+    # Write the result to a JSON file
+    with open(json_file_path, 'w', encoding='utf-8') as json_file:
+        json_file.write(result_img)
+
+    return json_file_path
+
+
+def process_folder_images_sequentially(folder_path, json_key_path):
+    """
+    Processes all images in the specified folder sequentially.
+
+    :param folder_path: Path to the folder containing images
+    :param json_key_path: Path to the JSON key for Vertex AI authentication
+    """
+    # Initialize Vertex AI client
+    client = VertexAIService(json_key_path=json_key_path)
+
+    # Get a list of all images in the folder
+    image_files = [os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith(('.jpg', '.jpeg', '.png'))]
+
+    # Process each image file sequentially
+    for file_path in image_files:
+        try:
+            json_file_path = process_image_file(file_path, client)
+            print(f"Processed: {file_path}, result saved to: {json_file_path}")
+            time.sleep(60)
+        except Exception as exc:
+            print(f"Failed to process file {file_path}. Error: {exc}")
+
+
+# Call the function to process the folder
+if __name__ == '__main__':
+    folder_path = './examples'  # Set the path to your folder
+    json_key_path = 'GOOGLE_VERTEX_AI_KEY.json'  # Set the path to your JSON key
+    process_folder_images_sequentially(folder_path, json_key_path)

prompt_v1.txt

@@ -0,0 +1,49 @@
+Task: Extract structured information from receipt images.
+Receipts: May be in various languages, including non-Latin scripts, and have diverse formats.
+Information to Extract:
+- Store name
+- Store address
+- Currency (e.g., USD, EUR)
+- date (in format "YYYY.MM.DD HH:MM:SS")
+- tax
+- Purchased items:
+	- Name
+	- Price per unit (format: 0.00)
+	- Quantity or weight
+	- unit of measurement (string)
+	- Total price (format: 0.00)
+	- Discount applied (if any, format: 0.00)
+	- Category (e.g., groceries, electronics, dining) — determine based on item name / description or context
+- Total amount (format: 0.00)
+- Total discount (format: 0.00)
+- Tax
+- Tips (if tips included in Total you need to write tips price, but if tips not included in Total you should write it's price as 0.00)
+Output Format: JSON object with the following structure:
+{
+  "store_name": "Store Name",
+  "store_address": "Store Address",
+  "currency": "Currency",
+  "date_time": "YYYY.MM.DD HH:MM:SS",
+  "payment_method": "card" or "cash
+  "items": [
+    {
+      "name": "Item Name",
+      "unit_price": 0.00,
+      "quantity": 0,
+      "unit_of_measurement":
+      "total_price": 0.00,
+      "discount": 0.00,
+      "category": "Category Name"
+    }
+  ],
+  "total_amount": 0.00,
+  "total_discount": 0.00,
+  "tax": 0.00
+  "tips": 0.00
+}
+If no receipt is detected: Return "Receipt not found."
+Additional Notes:
+1. If the receipt is in a non-Latin script, extract the information in its original form unless translation is required.
+2. If any information is unclear or missing, include it as "unknown" or "not available" in the output.
+Write whole json with information about all products.
+3. Note that the name of an item may be a few lines long, but the item necessarily has a price in last line.

prompt_v2.txt

@@ -0,0 +1,64 @@
+Task: Extract structured information from receipt images.
+Receipts: May be in various languages, including non-Latin scripts, and have diverse formats.
+Information to Extract:
+- Store name
+- Store address
+- Currency (e.g., USD, EUR)
+- date (in format "YYYY.MM.DD HH:MM:SS")
+- tax
+- tips (if tips included in Total you need to write tips price, but if tips not included in Total you should write it's price as 0.00)
+- Purchased items:
+	- Name
+	- Price per unit (format: 0.00)
+	- Quantity or weight
+	- unit of measurement (string)
+	- Total price (format: 0.00)
+	- Discount applied (if any, format: 0.00)
+	- Category (from the following list):
+		* Groceries
+			* Produce
+			* Meat
+			* Seafood
+			* Dairy
+			* Bakery
+			* Canned goods
+			* Frozen foods
+			* Beverages
+			* Snacks
+			* Cleaning supplies
+			* Personal care products
+		* Electronics
+		* Clothing
+		* Dining
+		* Home goods
+		* Other (specify if not in the list)
+- Total amount (format: 0.00)
+- Total discount (format: 0.00)
+Output Format: JSON object with the following structure:
+{
+  "store_name": "Store Name",
+  "store_address": "Store Address",
+  "currency": "Currency",
+  "date_time": "YYYY.MM.DD HH:MM:SS",
+  "payment_method": "card" or "cash
+  "items": [
+    {
+      "name": "Item Name",
+      "unit_price": 0.00,
+      "quantity": 0,
+      "unit_of_measurement":
+      "total_price": 0.00,
+      "discount": 0.00,
+      "category": "Category Name"
+    }
+  ],
+  "total_amount": 0.00,
+  "total_discount": 0.00,
+  "tax": 0.00,
+  "tips": 0.00
+}
+If no receipt is detected: Return "Receipt not found."
+Additional Notes:
+1. If the receipt is in a non-Latin script, extract the information in its original form unless translation is required.
+2. If any information is unclear or missing, include it as "unknown" or "not available" in the output.
+Write whole json with information about all products.

prompt_v3.txt

@@ -0,0 +1,116 @@
+#Your Task: Receipt Recognition and Data Extraction
+
+You are tasked with extracting structured information from receipts. Receipts will come from various countries, in different languages, and can have different layouts and formats. Your goal is to parse the receipt text, identify the receipt type (store, cafe/restaurant, or payment for services), and return the data in JSON format with the required fields. Follow the specific instructions below to ensure accurate extraction.
+
+#Required Fields:
+
+1. Receipt Type: Identify the type of receipt. It could be from:
+	* Store: Typically involves grocery or retail items.
+	* Cafe/Restaurant: Typically involves food and beverage items, table numbers, or tipping sections.
+	* Payment for Services: This type of receipt may involve service fees or professional services.
+2. Receipt Number: Extract the unique receipt number, typically found at the top of the receipt.
+3. Store/Business Name: Extract the name of the store, cafe, restaurant, or service provider.
+4. Store Address: Extract the address of the store, including city and country if available.
+5. Date: Extract the date of the transaction and format it as YYYY-MM-DD HH:MM.
+6. Currency: Extract the currency if explicitly mentioned (e.g., EUR, USD). If the currency is not specified, detect the language of the receipt and infer the currency based on the country where the language is predominantly used. For example, if the receipt is in Ukrainian, set the currency to UAH (Ukrainian Hryvnia).
+7. Payment Method: Identify whether the payment was made by "card" or "cash."
+8. Total Amount: Extract the total amount of the transaction. This is typically located at the end of the receipt, often highlighted in bold or a larger font.
+9. Total Discount: Extract the total discount if explicitly mentioned. If not, calculate the total discount by summing up the discounts for individual items.
+10. Tax: Extract the total tax amount if it is listed on the receipt.
+
+#Item-Level Details:
+
+For each item on the receipt, extract the following details:
+
+1. Item Name: Extract the full name of each item. Some items may have names split across multiple lines; in this case, concatenate the lines until you encounter a quantity or unit of measurement (e.g., "2ks"), which marks the end of the item name or on the next line. You should extract full name till statements as, for example, "1 ks" or "1 ks * 2"
+2. Unit Price: Extract the price per unit for each item.
+3. Quantity: Extract the quantity of each item, including the unit of measurement (e.g., "ks" for pieces, "kg" for kilograms).
+4. Price: Extract the final price for each item.
+5. Discount: Extract any discount applied to the item. If no discount is provided, set it to 0.
+6. Category: Automatically assign a category based on the item name. For groceries, assign relevant subcategories such as Dairy, Bakery, Fruits, etc. If this receipt was from restaurant - you should put category only from this list: Food, Drinks.
+
+#Special Cases:
+
+1. Cafe/Restaurant Receipts: If the receipt is from a cafe or restaurant, handle additional fields like:
+	* Table Number: Extract the table number if available, often printed near the top of the receipt.
+	* Tips: Extract any tip amounts explicitly listed or infer from the total paid amount minus the original bill amount.
+	* Service Charges: Some restaurants may include an automatic service charge, which should be listed separately from the tax or tips.
+	* Order Type: Identify whether the order was "dine-in" or "takeaway."
+2. Missing Currency: If no currency is mentioned on the receipt, infer the local currency by detecting the language and country of origin. For example, a receipt in French would use EUR, while one in Ukrainian would use UAH.
+3. Multi-line Item Names: If an item name spans multiple lines, merge the lines to form the complete name. Stop merging when a quantity or unit of measurement is encountered.
+4. Total Amount: The total amount is often larger than other numbers or displayed in bold at the bottom of the receipt. Make sure to capture this accurately.
+5. Total Discount: If no total discount is listed, sum the discounts for each individual item.
+6. Rounding Adjustments: Some receipts may include a "rounding" line item, where the total amount is adjusted (typically for cash payments) to avoid dealing with fractions of currency (e.g., rounding to the nearest 0.05 in some countries). If a rounding adjustment is present, extract the value of the rounding adjustment and reflect it in the total amount. For example:
+	* Total Before Rounding: 19.97
+	* Rounding: -0.02
+	* Final Total: 19.95 If the rounding adjustment is found, include it as a separate field in the JSON output under "rounding_adjustment", and ensure that the "total_amount" reflects the final adjusted total.
+7. Taxes: Receipts can handle taxes in various ways, and the system should be prepared to capture these scenarios:
+	* Tax-Inclusive Pricing: In some countries or for certain receipts, taxes are already included in the item price and not listed separately. If the receipt mentions that taxes are included in prices, record the "tax" field as 0 and note that taxes are included in the item prices.
+	* Multiple Tax Rates: Some receipts may include multiple tax rates (e.g., different VAT rates for different items). In this case, extract each tax rate and the corresponding tax amounts, and store them in a separate list of tax breakdowns. For example, the receipt might show "5% VAT" and "15% VAT" for different categories of goods:
+		** "taxes": [{"rate": "5%", "amount": 1.00}, {"rate": "15%", "amount": 3.50}]
+	* Missing Tax Information: In some cases, the receipt might not clearly mention taxes, but you may infer them based on standard rates in the country of origin. If no explicit tax amount is listed and you are unable to infer it, set the tax to "unknown" or null in the JSON output.
+	* Tax-Exempt Items: Some items on the receipt may be tax-exempt. If this is indicated, ensure that these items are excluded from any tax calculations. Note these in the item-level details with "tax_exempt": true and make sure the "tax" field reflects the correct amount for taxable items only.
+	* Service Charges vs. Taxes: Sometimes service charges may be listed separately from taxes (common in restaurants). Ensure that service charges are not included in the tax amount, and store them under the "service_charge" field.
+	* Tax Breakdown and Total: If both individual item taxes and a total tax amount are listed, the system should ensure consistency between the sum of item-level taxes and the total tax listed at the bottom of the receipt.
+8. In certain receipt formats, the quantity and unit price may appear before the item name. When processing such receipts, the goal is to correctly extract the quantity, unit price, and item name in their proper order. For example, if one line of the receipt shows "5 * 23.00 = 115.0" and the next line displays "Milk," the system should interpret this as:
+	* Quantity: 5 units
+	* Unit Price: 23.00
+	* Item Name: Milk
+	* Total Price: 115.0 This approach should be applied consistently throughout the entire receipt to extract data accurately.
+
+#JSON Output Format:
+
+{
+  "receipt_type": "string",
+  "receipt_number": "string",
+  "store_name": "string",
+  "store_address": "string",
+  "date_time": "string",
+  "currency": "string",
+  "payment_method": "string",
+  "total_amount": "number",
+  "total_discount": "number",
+  "tax": "number",
+  "taxes": [
+    {
+      "rate": "string",
+      "amount": "number"
+    }
+  ],
+  "rounding_adjustment": "number",
+  "rounded_total_aount": "number",
+  "items": [
+    {
+      "name": "string",
+      "unit_price": "number",
+      "quantity": {
+        "amount": "number",
+        "unit_of_measurement": "string"
+      },
+      "price": "number",
+      "discount": "number",
+      "category": "string",
+      "tax_exempt": "boolean"
+    }
+  ],
+  "cafe_additional_info": {
+    "table_number": "string",
+    "tips": "number",
+    "service_charge": "number",
+    "order_type": "string"
+  }
+}
+
+#Additional Notes:
+
+1. You should handle receipts in various languages and from different countries.
+2. Pay special attention to formatting differences and edge cases, such as multi-line item names, missing currency symbols, or cafe/restaurant-specific information.
+3. Always ensure the output is well-structured and follows the JSON format provided.
+4. The "rounding_adjustment" field should reflect the value by which the total was adjusted due to rounding. If no rounding adjustment is present, it can be set to 0 or omitted from the output.
+5. Ensure that the final "total_amount" field reflects the total after any rounding adjustment has been applied.
+6. Inclusive Taxes: If taxes are included in the item prices, set the "tax" field to 0 and adjust the item prices accordingly.
+7. Multiple Tax Rates: The "taxes" field provides a detailed breakdown for receipts with different tax rates. This field is optional and can be excluded if only a single tax amount is listed.
+8. Tax-Exempt Items: Mark tax-exempt items with the "tax_exempt": true field.
+9. Service Charges vs. Taxes: Ensure that service charges are captured separately from taxes in the "service_charge" field.
+10. Return the full JSON object with all available information. If any information is unclear or missing, include it as "unknown" or "not available" in the output.
+11. Your final response should be in valid JSON format with no additional text.

requirements.txt

@@ -0,0 +1,15 @@
+torch==2.4.1
+torchvision==0.19.1
+gradio==4.44.0
+numpy==1.26.4
+opencv-python==4.9.0.80
+mediapipe==0.10.7
+openai==1.45.0
+google-api-python-client==2.145.0
+google-auth-httplib2==0.2.0
+google-auth-oauthlib==1.2.1
+google-cloud-aiplatform==1.67.0
+aiohttp==3.10.5
+openpyxl==3.1.5
+torch==2.4.1
+torchvision==0.19.1

system_instruction.txt

@@ -0,0 +1 @@
+You are receipts recognizer

utils.py

@@ -0,0 +1,170 @@
+import numpy as np
+import json
+import pandas as pd
+
+import base64
+from io import BytesIO
+from PIL import Image
+from algorithm import receipt_calculation
+
+
+def read_prompt_from_file(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        return file.read()
+
+
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+
+
+def encode_image_from_gradio(input_image):
+    if input_image is None:
+        return None
+
+    pil_image = Image.fromarray(np.uint8(input_image))
+
+    buffered = BytesIO()
+    pil_image.save(buffered, format="JPEG")
+
+    base64_image = base64.b64encode(buffered.getvalue()).decode('utf-8')
+    return base64_image
+
+
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+
+
+def encode_image_to_jpeg_base64(filepath):
+    """
+    Encodes an image file to JPEG and then to a Base64 string.
+
+    :param filepath: Path to the image file
+    :return: Base64 encoded string of the JPEG image or an error message
+    """
+    if filepath is None:
+        return None, "File path is None."
+
+    try:
+        pil_image = Image.open(filepath)
+
+        if pil_image.mode == 'RGBA':
+            pil_image = pil_image.convert('RGB')
+
+        buffered = BytesIO()
+        pil_image.save(buffered, format="JPEG")
+
+        base64_image = base64.b64encode(buffered.getvalue()).decode('utf-8')
+        return base64_image
+
+    except Exception as e:
+        return None, str(e)
+
+
+def encode_image_to_webp_base64(filepath):
+    """
+    Encodes an image file to JPEG and then to a Base64 string.
+
+    :param filepath: Path to the image file
+    :return: Base64 encoded string of the JPEG image or an error message
+    """
+    if filepath is None:
+        return None, "File path is None."
+
+    try:
+        pil_image = Image.open(filepath)
+
+        if pil_image.mode == 'RGBA':
+            pil_image = pil_image.convert('RGB')
+
+        buffered = BytesIO()
+        pil_image.save(buffered, format="WEBP")
+
+        base64_image = base64.b64encode(buffered.getvalue()).decode('utf-8')
+        return base64_image
+
+    except Exception as e:
+        return None, str(e)
+
+
+def process_receipt_json(json_input):
+    try:
+        # Parse the JSON string
+        data = json.loads(json_input)
+    except json.JSONDecodeError:
+        return "Error: Invalid JSON format", None
+
+    # Initialize the output information
+    store_info = ""
+    items_table = []
+
+    # Try to extract store information, if available
+    store_name = data.get("store_name", "Not specified")
+    store_address = data.get("store_address", "Not specified")
+    currency = data.get("currency", "Not specified")
+    date_time = data.get("date_time", "Not specified")
+    type = data.get("payment_method", "Not specified")
+    total_amount = data.get("total_amount", "Not specified")
+    discount = data.get("total_discount", "Not specified")
+    tax = data.get("tax", "Not specified")
+    tips = data.get("tips", "Not specified")
+
+    # Format store information
+    store_info = \
+        (f"Store: {store_name}\nAddress: {store_address}\nCurrency: {currency}"
+         f"\nTotal Amount: {total_amount}"
+         f"\nDate time: {date_time}"
+         f"\nType payment: {type}"
+         f"\nTotal discount: {discount}"
+         f"\nTax: {tax}"
+         )
+
+    # Extract items, if available in JSON
+    items = data.get("items", [])
+    if items:
+        items_table = [[item.get("name", "Not specified"),
+                        item.get("category", "Not specified"),
+                        item.get("unit_price", "Not specified"),
+                        item.get("quantity", "Not specified"),
+                        item.get("unit_of_measurement", "Not specified"),
+                        item.get("total_price", "Not specified"),
+                        item.get("discount", "Not specified")]
+                        for item in items]
+    else:
+        items_table = [["No items"]]
+
+    total_product_prices, total_sum = receipt_calculation.calculate_tips_and_taxes(items_table, total_amount, tax, tips)
+    message = "Everything is okay!"
+    if items_table[0][0] != "No items":
+        for i in range(len(items_table)):
+            items_table[i].append(total_product_prices[i].price)
+
+    if total_sum != round(float(str(total_amount).replace(",", ".")), 2):
+        message = "Recognized total sum and products total sum is not equal. Check if AI model correctly created a JSON"
+    return store_info, items_table, message
+
+
+def save_to_excel(json_output, excel_file_path):
+    store_info, items_table, _ = process_receipt_json(json_output)
+    if isinstance(store_info, str) and store_info.startswith("Error:"):
+        return store_info
+
+    store_info_lines = store_info.split('\n')
+    store_info_parsed = [(line.split(': ')[0], line.split(': ')[1]) for line in store_info_lines if ': ' in line]
+
+    store_info_df = pd.DataFrame(store_info_parsed, columns=["Label", "Value"])
+
+    items_df = pd.DataFrame(items_table, columns=[
+        "Item Name", "Category", "Unit Price", "Quantity", "Unit", "Total Price", "Discount", "Grand Total"
+    ])
+
+    with pd.ExcelWriter(excel_file_path, engine='openpyxl') as writer:
+        store_info_df.to_excel(writer, sheet_name='Receipt', index=False, header=False, startrow=0)
+
+        items_df_start_row = len(store_info_df) + 2
+        items_df.to_excel(writer, sheet_name='Receipt', index=False, startrow=items_df_start_row)
+        worksheet = writer.sheets['Receipt']
+        worksheet.column_dimensions['A'].width = 30
+
+    return excel_file_path

vertex_ai_example.py

@@ -0,0 +1,20 @@
+import vertexai
+from google.cloud import aiplatform
+from google.oauth2 import service_account
+from vertexai.generative_models import GenerativeModel
+
+service_account_path = "receiptsai19_09.json"
+
+credentials = service_account.Credentials.from_service_account_file(service_account_path)
+
+def generate():
+    vertexai.init(project="receiptsai-436007", location="us-central1", credentials=credentials)
+    model = GenerativeModel(
+        "gemini-1.5-flash-001"
+    )
+    responses = model.generate_content('Create simple example on C++')
+
+    print(responses.text)
+
+
+generate()

vertex_ai_service.py

@@ -0,0 +1,168 @@
+import base64
+import json
+import os
+import tempfile
+
+import vertexai
+from google.oauth2 import service_account
+from vertexai.generative_models import GenerativeModel
+from vertexai.generative_models import Part, SafetySetting
+
+
+class VertexAIService:
+    def __init__(self, json_key_path=None, json_key_env_var='GOOGLE_VERTEX_KEY'):
+        """
+        Initializes the Vertex AI client.
+
+        :param json_key_path: Path to the JSON file (optional)
+        :param json_key_env_var: Environment variable name containing the JSON key (default 'GOOGLE_VERTEX_KEY')
+        """
+        self.json_key_path = json_key_path
+        self.json_key_env_var = json_key_env_var
+        self.credentials = self._authenticate_vertex_ai()
+
+    def _authenticate_vertex_ai(self):
+        """
+        Authenticates using the JSON key from a file or environment variable.
+
+        :return: Google Credentials object
+        """
+        if self.json_key_path and os.path.isfile(self.json_key_path):
+            # Authenticate using the file
+            creds = service_account.Credentials.from_service_account_file(self.json_key_path)
+        else:
+            # Get JSON key from environment variable
+            json_key = os.getenv(self.json_key_env_var)
+
+            if not json_key:
+                raise ValueError(f"Environment variable {self.json_key_env_var} is not set.")
+
+            # Save JSON key to a temporary file
+            with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+                temp_file.write(json_key.encode('utf-8'))
+                temp_file_path = temp_file.name
+
+            # Create credentials object
+            creds = service_account.Credentials.from_service_account_file(temp_file_path)
+
+            # Remove the temporary file after creating the credentials object
+            os.remove(temp_file_path)
+
+        return creds
+
+    def process_text(self, project, location, model_name, prompt, system=""):
+        """
+        Generates content using Vertex AI.
+
+        :param project: GCP project ID
+        :param location: GCP region
+        :param model_name: The name of the Generative Model
+        :param prompt: Prompt for content generation
+        :return: Generated content or error message
+        """
+        try:
+            # Initialize Vertex AI with credentials
+            vertexai.init(project=project, location=location, credentials=self.credentials)
+            model = GenerativeModel(model_name, system_instruction=[system])
+
+            # Generate content
+            responses = model.generate_content(prompt)
+            return responses.text
+        except Exception as error:
+            return f"An error occurred: {error}"
+
+    def process_image(self, input_image64, model_name, prompt, system="You are receipt recognizer", temperatura=0.0):
+        """
+        Processes the image using Vertex AI model.
+
+        :param input_image64: Base64 encoded image string
+        :param model_name: Name of the model in Vertex AI
+        :param prompt: Text prompt to guide the model
+        :param temperatura: Temperature for controlling randomness
+        :return: JSON response or error message
+        """
+        if input_image64 is None:
+            raise ValueError("No objects detected.")
+
+        try:
+            # Initialize Vertex AI
+            vertexai.init(project="receiptsai-436007", location="us-central1", credentials=self.credentials)
+
+            # Load the model
+            model = GenerativeModel(model_name, system_instruction=[system])
+
+            # Prepare the image part
+            image_part = Part.from_data(
+                mime_type="image/webp",
+                data=input_image64
+            )
+
+            # Set generation configuration
+            generation_config = {
+                "max_output_tokens": 8192,
+                "temperature": temperatura,
+                "response_mime_type": "application/json"
+            }
+
+            # Set safety settings
+            safety_settings = [
+                SafetySetting(
+                    category=SafetySetting.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+                    threshold=SafetySetting.HarmBlockThreshold.BLOCK_NONE
+                ),
+                SafetySetting(
+                    category=SafetySetting.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+                    threshold=SafetySetting.HarmBlockThreshold.BLOCK_NONE
+                ),
+                SafetySetting(
+                    category=SafetySetting.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+                    threshold=SafetySetting.HarmBlockThreshold.BLOCK_NONE
+                ),
+                SafetySetting(
+                    category=SafetySetting.HarmCategory.HARM_CATEGORY_HARASSMENT,
+                    threshold=SafetySetting.HarmBlockThreshold.BLOCK_NONE
+                ),
+            ]
+
+            # Generate content using the model
+            response = model.generate_content([image_part, prompt],
+                                              generation_config=generation_config,
+                                              safety_settings=safety_settings)
+
+            json_content = json.loads(response.text)
+
+            json_content['input_tokens'] = response.usage_metadata.prompt_token_count
+            json_content['output_tokens'] = response.usage_metadata.candidates_token_count
+            json_content['total_tokens'] = response.usage_metadata.total_token_count
+
+            return json.dumps(json_content, indent=4)
+
+        except Exception as error:
+            raise RuntimeError(f"An error occurred during image processing: {error}")
+
+
+# Example usage
+if __name__ == '__main__':
+    # Project and model details
+    project = "receiptsai-436007"
+    location = "us-central1"
+    model_name = "gemini-1.5-flash-001"
+    prompt = "Create example on C++"
+
+    # Initialize the client and generate content
+    client = VertexAIService(json_key_path='GOOGLE_VERTEX_AI_KEY.json')
+    result = client.process_text(project, location, model_name, prompt)
+    print(f'Generated result: {result}')
+
+    # Image processing
+    image_path = "./examples/lidl2.jpg"
+    with open(image_path, "rb") as image_file:
+        input_image64 = base64.b64encode(image_file.read()).decode('utf-8')
+
+    prompt = "Read the text"
+    system = "You are receipt recognizer"
+    result_img = client.process_image(input_image64, "gemini-1.5-pro", prompt, system, 0.0)
+    print(f'Image processing result: {result_img}')
+    decoded_string = result_img.encode('utf-8').decode('unicode_escape')
+
+    print(decoded_string)