Spaces:
Sleeping
Sleeping
valentynliubchenko
commited on
Commit
·
eba303d
1
Parent(s):
75a9c52
merging
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- .env +3 -0
- .gitattributes +7 -0
- algorithm/receipt_calculation.py +4 -3
- app.py +122 -54
- common/enum/__init__.py +0 -0
- common/enum/ai_service_error.py +6 -0
- common/enum/payment_method.py +7 -0
- common/exceptions.py +20 -0
- prompt_v1.txt → common/prompt_v1.txt +35 -26
- prompt_v2.txt → common/prompt_v2.txt +0 -0
- common/prompt_v3.txt +116 -0
- common/utils/__init__.py +38 -0
- common/utils/utils.py +94 -0
- convert_to_webp.py +1 -1
- examples/5404832557079585491.webp +0 -3
- examples/ATB_11.webp +0 -3
- examples/Sportisimo.webp +0 -3
- examples/cafe.webp +0 -3
- examples/farmacy_1.webp +0 -3
- examples/farmacy_2.webp +0 -3
- examples/fatlouis.webp +0 -3
- examples/garm_3.webp +0 -3
- examples/image_2024_09_25T12_13_01_811Z.webp +0 -3
- examples/lidl1.webp +0 -3
- examples/lidl2.webp +0 -3
- examples/photo_2024-09-10_10-06-14.webp +0 -3
- examples/photo_2024-09-10_10-06-24.webp +0 -3
- examples/photo_2024-09-10_10-06-28.webp +0 -3
- examples/photo_2024-09-24_14-50-34.webp +0 -3
- examples/tiket.webp +0 -3
- examples_canada/photo_2024-10-09_14-21-54.webp +0 -0
- examples_canada/photo_2024-10-09_14-23-03.webp +0 -0
- examples_canada/photo_2024-10-10_15-23-22.webp +0 -0
- examples_france/photo_1_2024-10-02_00-08-53.webp +0 -0
- examples_france/photo_2024-10-07_21-46-05.webp +0 -0
- examples_france/photo_2024-10-07_21-46-27.webp +0 -0
- examples_france/photo_2_2024-10-02_00-08-53.webp +0 -0
- examples_france/photo_3_2024-10-02_00-08-53.webp +0 -0
- examples_france/photo_4_2024-10-02_00-08-53.webp +0 -0
- examples_france/photo_5_2024-10-02_00-08-53.webp +0 -0
- examples_france/photo_6_2024-10-02_00-08-53.webp +0 -0
- examples_france/photo_7_2024-10-02_00-08-53.webp +0 -0
- examples_france/photo_8_2024-10-02_00-08-53.webp +0 -0
- examples_us/Photo Sep 29 2024, 15 43 17.webp +0 -0
- examples_us/Photo Sep 29 2024, 15 43 30.webp +0 -0
- examples_us/Photo Sep 29 2024, 15 43 48.webp +0 -0
- examples_us/Photo Sep 29 2024, 15 44 21.webp +0 -0
- examples_us/Photo Sep 29 2024, 15 44 42.webp +0 -0
- examples_us/Photo Sep 29 2024, 15 45 13.webp +0 -0
- examples_us/Photo Sep 29 2024, 15 45 26.webp +0 -0
.env
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
COLLECTION_DATA_VERSION=True
|
| 2 |
+
|
| 3 |
+
|
.gitattributes
CHANGED
|
@@ -35,4 +35,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
examples/*.JPG filter=lfs diff=lfs merge=lfs -text
|
| 37 |
examples/*.jpg filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 38 |
examples/*.* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
examples/*.JPG filter=lfs diff=lfs merge=lfs -text
|
| 37 |
examples/*.jpg filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
examples/*.webp filter=lfs diff=lfs merge=lfs -text
|
| 39 |
examples/*.* filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
examples_sl/*.* filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
examples_ua/*.* filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
examples_us/*.* filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
examples_canada/*.* filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
examples_france/*.* filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
|
algorithm/receipt_calculation.py
CHANGED
|
@@ -88,6 +88,7 @@ def second_algorithm(_products_total, receipt_total):
|
|
| 88 |
|
| 89 |
|
| 90 |
def clean_and_convert_to_float(price):
|
|
|
|
| 91 |
clean_price = ''.join(c for c in str(price) if c.isdigit() or c in ",.")
|
| 92 |
return float(clean_price.replace(",", "."))
|
| 93 |
|
|
@@ -119,9 +120,9 @@ def calculate_tips_and_taxes(items_table, total_amount, tax, tips):
|
|
| 119 |
sum_of_product_prices += _product.price
|
| 120 |
|
| 121 |
sum_of_product_prices = round(float(sum_of_product_prices), 2)
|
| 122 |
-
total_amount = round(
|
| 123 |
-
tips = round(
|
| 124 |
-
tax = round(tips + round(
|
| 125 |
if round(float(total_amount), 2) != round(float(sum_of_product_prices) + float(tax), 2):
|
| 126 |
return products, sum_of_product_prices
|
| 127 |
|
|
|
|
| 88 |
|
| 89 |
|
| 90 |
def clean_and_convert_to_float(price):
|
| 91 |
+
if price == "": return 0.0
|
| 92 |
clean_price = ''.join(c for c in str(price) if c.isdigit() or c in ",.")
|
| 93 |
return float(clean_price.replace(",", "."))
|
| 94 |
|
|
|
|
| 120 |
sum_of_product_prices += _product.price
|
| 121 |
|
| 122 |
sum_of_product_prices = round(float(sum_of_product_prices), 2)
|
| 123 |
+
total_amount = round(clean_and_convert_to_float(total_amount), 2)
|
| 124 |
+
tips = round(clean_and_convert_to_float(tips), 2)
|
| 125 |
+
tax = round(tips + round(clean_and_convert_to_float(tax), 2), 2)
|
| 126 |
if round(float(total_amount), 2) != round(float(sum_of_product_prices) + float(tax), 2):
|
| 127 |
return products, sum_of_product_prices
|
| 128 |
|
app.py
CHANGED
|
@@ -1,24 +1,35 @@
|
|
| 1 |
import json
|
| 2 |
-
import pandas as pd
|
| 3 |
import os
|
| 4 |
from datetime import datetime
|
| 5 |
|
| 6 |
import gradio as gr
|
| 7 |
from PIL import Image
|
|
|
|
| 8 |
|
| 9 |
from google_drive_client import GoogleDriveClient
|
| 10 |
from openai_service import OpenAIService
|
| 11 |
-
from
|
|
|
|
|
|
|
| 12 |
from vertex_ai_service import VertexAIService
|
| 13 |
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
prompt_names = ["prompt_v1", "prompt_v2", "prompt_v3"]
|
| 16 |
-
example_list = [["./examples/" + example] for example in os.listdir("examples")]
|
|
|
|
|
|
|
| 17 |
example_list_us = [["./examples_us/" + example] for example in os.listdir("examples_us")]
|
| 18 |
example_list_canada = [["./examples_canada/" + example] for example in os.listdir("examples_canada")]
|
| 19 |
example_france = [["./examples_france/" + example] for example in os.listdir("examples_france")]
|
| 20 |
|
| 21 |
-
prompt_default = read_prompt_from_file("prompt_v1.txt")
|
| 22 |
system_instruction = read_prompt_from_file("system_instruction.txt")
|
| 23 |
|
| 24 |
|
|
@@ -30,9 +41,9 @@ def process_image(input_image, model_name, prompt_name, temperatura, system_inst
|
|
| 30 |
if system_instruction is None:
|
| 31 |
system_instruction = ""
|
| 32 |
if input_image is None:
|
| 33 |
-
return
|
| 34 |
-
|
| 35 |
-
|
| 36 |
if prompt_name is None:
|
| 37 |
prompt_name = "prompt_v1"
|
| 38 |
prompt_file = f"{prompt_name}.txt"
|
|
@@ -57,30 +68,72 @@ def process_image(input_image, model_name, prompt_name, temperatura, system_inst
|
|
| 57 |
base64_image = encode_image_to_webp_base64(input_image)
|
| 58 |
|
| 59 |
try:
|
| 60 |
-
if model_name.startswith("
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
# result = gpt_process_image(base64_image, model_name, prompt, system_instruction, temperatura)
|
| 62 |
-
result = open_ai_client.process_image(base64_image, model_name, prompt, system_instruction, temperatura)
|
|
|
|
|
|
|
| 63 |
else:
|
| 64 |
-
result = vertex_ai_client.process_image(base64_image, model_name, prompt, system_instruction,
|
| 65 |
temperatura)
|
| 66 |
-
|
|
|
|
|
|
|
|
|
|
| 67 |
result = json.dumps(parsed_result, ensure_ascii=False, indent=4)
|
| 68 |
# result = result.encode('utf-8').decode('unicode_escape')
|
| 69 |
print(result)
|
| 70 |
except Exception as e:
|
| 71 |
print(f"Exception occurred: {e}")
|
| 72 |
result = json.dumps({"error": "Error processing: Check prompt or images"})
|
|
|
|
|
|
|
| 73 |
|
| 74 |
# print (result)
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
|
| 79 |
-
return model_name, result, store_info, items_table, message, gr.update(interactive=True),
|
|
|
|
| 80 |
|
| 81 |
|
| 82 |
-
def
|
| 83 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
save_button_update = gr.update(interactive=False)
|
| 85 |
image_link, json_link, excel_link = None, None, None
|
| 86 |
try:
|
|
@@ -144,13 +197,16 @@ def save_flag_data(save_type, image, model_name, prompt_name, temperatura, curre
|
|
| 144 |
json_file.write(data_to_save_encode)
|
| 145 |
|
| 146 |
excel_file_path = os.path.join(flagging_dir, f"{base_filename}.xlsx")
|
| 147 |
-
|
|
|
|
|
|
|
|
|
|
| 148 |
|
| 149 |
# Upload files to Google Drive
|
| 150 |
-
google_drive_client_current = GoogleDriveClient(json_key_path='GOOGLE_SERVICE_ACCOUNT_KEY.json')
|
| 151 |
if google_drive_client_current:
|
| 152 |
try:
|
| 153 |
-
image_folder_id = '
|
| 154 |
image_link = google_drive_client_current.upload_file(image_save_path, image_folder_id)
|
| 155 |
json_link = google_drive_client_current.upload_file(json_file_path, image_folder_id)
|
| 156 |
excel_link = google_drive_client_current.upload_file(excel_file_path, image_folder_id)
|
|
@@ -164,24 +220,26 @@ def save_flag_data(save_type, image, model_name, prompt_name, temperatura, curre
|
|
| 164 |
|
| 165 |
except Exception as e:
|
| 166 |
print(f"Error while saving flag data: {e}")
|
| 167 |
-
links = f"Image: {image_link}\nJSON: {json_link}\nExcel: {excel_link}"
|
| 168 |
return save_button_update, save_button_update, save_button_update, links
|
| 169 |
|
|
|
|
| 170 |
def update_prompt_from_radio(prompt_name):
|
| 171 |
if prompt_name == "prompt_v1":
|
| 172 |
-
return read_prompt_from_file("prompt_v1.txt")
|
| 173 |
elif prompt_name == "prompt_v2":
|
| 174 |
-
return read_prompt_from_file("prompt_v2.txt")
|
| 175 |
elif prompt_name == "prompt_v3":
|
| 176 |
-
return read_prompt_from_file("prompt_v3.txt")
|
| 177 |
else:
|
| 178 |
-
return read_prompt_from_file("prompt_v1.txt")
|
| 179 |
|
| 180 |
-
|
| 181 |
-
|
|
|
|
| 182 |
|
| 183 |
key = None
|
| 184 |
-
key_file_path = 'OPENAI_AI_KEY.txt'
|
| 185 |
if os.path.exists(key_file_path):
|
| 186 |
try:
|
| 187 |
with open(key_file_path, 'r') as key_file:
|
|
@@ -192,38 +250,43 @@ if os.path.exists(key_file_path):
|
|
| 192 |
open_ai_client = OpenAIService(api_key=key)
|
| 193 |
|
| 194 |
with gr.Blocks() as iface:
|
| 195 |
-
gr.Markdown("#
|
| 196 |
-
gr.Markdown("
|
| 197 |
|
| 198 |
with gr.Row():
|
| 199 |
with gr.Column(scale=1):
|
| 200 |
image_input = gr.Image(type="filepath")
|
| 201 |
-
model_radio = gr.Radio(model_names, label="Choose model", value=model_names[0])
|
| 202 |
-
prompt_radio = gr.Radio(prompt_names, label="Choose prompt", value=prompt_names[0])
|
| 203 |
-
temperature_slider = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label="Temperatura", value=0.0
|
| 204 |
-
|
| 205 |
-
|
|
|
|
| 206 |
with gr.Row():
|
| 207 |
-
submit_button = gr.Button("
|
| 208 |
|
| 209 |
with gr.Column(scale=2):
|
| 210 |
-
model_output = gr.Textbox(label="MODEL", lines=1, interactive=
|
| 211 |
json_output = gr.Textbox(label="Result as json")
|
| 212 |
store_info_output = gr.Textbox(label="Store Information", lines=4)
|
| 213 |
items_list = gr.Dataframe(
|
| 214 |
-
headers=["Item Name", "Category", "Unit Price", "Quantity", "Unit", "Total Price", "Discount",
|
|
|
|
| 215 |
label="Items List")
|
|
|
|
|
|
|
|
|
|
| 216 |
comments_output = gr.Textbox(label="Comments", visible=True, lines=4, interactive=True)
|
| 217 |
with gr.Row():
|
| 218 |
save_good_button = gr.Button(value="Save as Good", interactive=False)
|
| 219 |
-
save_average_button = gr.Button(value="Save as Average"
|
| 220 |
save_poor_button = gr.Button(value="Save as Poor", interactive=False)
|
| 221 |
file_links_output = gr.Textbox(label="File Links", interactive=False, visible=True)
|
| 222 |
submit_button.click(fn=process_image,
|
| 223 |
inputs=[image_input, model_radio, prompt_radio, temperature_slider, system_instruction,
|
| 224 |
custom_prompt],
|
| 225 |
-
outputs=[model_output, json_output, store_info_output, items_list, comments_output,
|
| 226 |
-
save_good_button, save_average_button, save_poor_button])
|
| 227 |
common_inputs = [image_input, model_radio, prompt_radio, temperature_slider, custom_prompt, model_output,
|
| 228 |
json_output, store_info_output, items_list, comments_output, system_instruction]
|
| 229 |
|
|
@@ -242,6 +305,7 @@ with gr.Blocks() as iface:
|
|
| 242 |
)
|
| 243 |
return save_good_update, save_avg_update, save_poor_update, file_links
|
| 244 |
|
|
|
|
| 245 |
# Use the same common_inputs for all buttons but ensure the correct values are passed
|
| 246 |
save_good_button.click(
|
| 247 |
fn=lambda *args: save_flag_data_wrapper("Good", *args),
|
|
@@ -261,20 +325,24 @@ with gr.Blocks() as iface:
|
|
| 261 |
outputs=[save_good_button, save_average_button, save_poor_button, file_links_output]
|
| 262 |
)
|
| 263 |
prompt_radio.change(fn=update_prompt_from_radio, inputs=[prompt_radio], outputs=[custom_prompt])
|
| 264 |
-
gr.Examples(examples=
|
| 265 |
inputs=[image_input, model_radio, prompt_radio, temperature_slider, custom_prompt],
|
| 266 |
-
label="Examples for
|
|
|
|
|
|
|
|
|
|
|
|
|
| 267 |
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
|
| 280 |
-
iface.launch()
|
|
|
|
| 1 |
import json
|
|
|
|
| 2 |
import os
|
| 3 |
from datetime import datetime
|
| 4 |
|
| 5 |
import gradio as gr
|
| 6 |
from PIL import Image
|
| 7 |
+
from dotenv import load_dotenv
|
| 8 |
|
| 9 |
from google_drive_client import GoogleDriveClient
|
| 10 |
from openai_service import OpenAIService
|
| 11 |
+
from qr_retriever import get_receipt_by_qr
|
| 12 |
+
from utils import read_prompt_from_file, process_receipt_json, save_to_excel, \
|
| 13 |
+
encode_image_to_webp_base64
|
| 14 |
from vertex_ai_service import VertexAIService
|
| 15 |
|
| 16 |
+
load_dotenv()
|
| 17 |
+
isFullVersion = os.getenv("COLLECTION_DATA_VERSION") != "True"
|
| 18 |
+
if isFullVersion:
|
| 19 |
+
model_names = ["gemini-1.5-flash", "gemini-1.5-pro", "gemini-flash-experimental", "gemini-pro-experimental",
|
| 20 |
+
"gpt-4o-mini", "gpt-4o", "QR-processing"]
|
| 21 |
+
else:
|
| 22 |
+
model_names = ["gemini-1.5-flash", "gemini-1.5-pro", "gemini-flash-experimental", "gemini-pro-experimental","QR-processing"]
|
| 23 |
+
|
| 24 |
prompt_names = ["prompt_v1", "prompt_v2", "prompt_v3"]
|
| 25 |
+
# example_list = [["./examples/" + example] for example in os.listdir("examples")]
|
| 26 |
+
example_list_sl = [["./examples_sl/" + example] for example in os.listdir("examples_sl")]
|
| 27 |
+
example_list_ua = [["./examples_ua/" + example] for example in os.listdir("examples_ua")]
|
| 28 |
example_list_us = [["./examples_us/" + example] for example in os.listdir("examples_us")]
|
| 29 |
example_list_canada = [["./examples_canada/" + example] for example in os.listdir("examples_canada")]
|
| 30 |
example_france = [["./examples_france/" + example] for example in os.listdir("examples_france")]
|
| 31 |
|
| 32 |
+
prompt_default = read_prompt_from_file("common/prompt_v1.txt")
|
| 33 |
system_instruction = read_prompt_from_file("system_instruction.txt")
|
| 34 |
|
| 35 |
|
|
|
|
| 41 |
if system_instruction is None:
|
| 42 |
system_instruction = ""
|
| 43 |
if input_image is None:
|
| 44 |
+
return model_name, "Image not found. Load image ", "", [], [], "", gr.update(interactive=False), gr.update(
|
| 45 |
+
interactive=False), gr.update(interactive=False), ""
|
| 46 |
+
|
| 47 |
if prompt_name is None:
|
| 48 |
prompt_name = "prompt_v1"
|
| 49 |
prompt_file = f"{prompt_name}.txt"
|
|
|
|
| 68 |
base64_image = encode_image_to_webp_base64(input_image)
|
| 69 |
|
| 70 |
try:
|
| 71 |
+
if model_name.startswith("QR"):
|
| 72 |
+
try:
|
| 73 |
+
original_json, parsed_result = get_receipt_by_qr(input_image)
|
| 74 |
+
except Exception as e:
|
| 75 |
+
print(e)
|
| 76 |
+
return model_name, "Error get_receipt_by_qr", "", [], [], "", gr.update(interactive=False), gr.update(
|
| 77 |
+
interactive=False), gr.update(interactive=False), ""
|
| 78 |
+
print("original_json", original_json)
|
| 79 |
+
print("receipt", parsed_result)
|
| 80 |
+
if parsed_result:
|
| 81 |
+
parsed_result = clean_value(parsed_result)
|
| 82 |
+
parsed_result["sub_total_amount"] = "unknown"
|
| 83 |
+
for key, value in parsed_result.items():
|
| 84 |
+
print(f"Key: {key}, Value: {value}")
|
| 85 |
+
|
| 86 |
+
elif model_name.startswith("gpt"):
|
| 87 |
# result = gpt_process_image(base64_image, model_name, prompt, system_instruction, temperatura)
|
| 88 |
+
result, model_input = open_ai_client.process_image(base64_image, model_name, prompt, system_instruction, temperatura)
|
| 89 |
+
parsed_result = json.loads(result)
|
| 90 |
+
|
| 91 |
else:
|
| 92 |
+
result, model_input = vertex_ai_client.process_image(base64_image, model_name, prompt, system_instruction,
|
| 93 |
temperatura)
|
| 94 |
+
parsed_result = json.loads(result)
|
| 95 |
+
|
| 96 |
+
parsed_result['file_name'] = os.path.basename(input_image)
|
| 97 |
+
|
| 98 |
result = json.dumps(parsed_result, ensure_ascii=False, indent=4)
|
| 99 |
# result = result.encode('utf-8').decode('unicode_escape')
|
| 100 |
print(result)
|
| 101 |
except Exception as e:
|
| 102 |
print(f"Exception occurred: {e}")
|
| 103 |
result = json.dumps({"error": "Error processing: Check prompt or images"})
|
| 104 |
+
return model_name, result, "", "", "", "", gr.update(interactive=True), gr.update(
|
| 105 |
+
interactive=True), gr.update(interactive=True), ""
|
| 106 |
|
| 107 |
# print (result)
|
| 108 |
+
try:
|
| 109 |
+
store_info, items_table, taxs_table, message = process_receipt_json(result)
|
| 110 |
+
print(store_info)
|
| 111 |
+
print(items_table)
|
| 112 |
+
except Exception as e:
|
| 113 |
+
print(f"Exception occurred: {e}")
|
| 114 |
+
result = json.dumps({"error": "process_receipt_json"})
|
| 115 |
+
return model_name, result, "", "", "", "", gr.update(interactive=False), gr.update(
|
| 116 |
+
interactive=False), gr.update(interactive=False), ""
|
| 117 |
|
| 118 |
+
return model_name, result, store_info, items_table, taxs_table, message, gr.update(interactive=True), gr.update(
|
| 119 |
+
interactive=True), gr.update(interactive=True), ""
|
| 120 |
|
| 121 |
|
| 122 |
+
def clean_value(value):
|
| 123 |
+
if isinstance(value, list):
|
| 124 |
+
return [clean_value(v) for v in value]
|
| 125 |
+
elif isinstance(value, dict):
|
| 126 |
+
return {k: clean_value(v) for k, v in value.items()}
|
| 127 |
+
elif value is None:
|
| 128 |
+
return "unknown"
|
| 129 |
+
else:
|
| 130 |
+
return value
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def save_flag_data(save_type, image, model_name, prompt_name, temperatura, current_prompt_text, model_output,
|
| 134 |
+
json_output,
|
| 135 |
+
store_info_output, items_list, comments_output, system_instruction,
|
| 136 |
+
flagging_dir="custom_flagged_data"):
|
| 137 |
save_button_update = gr.update(interactive=False)
|
| 138 |
image_link, json_link, excel_link = None, None, None
|
| 139 |
try:
|
|
|
|
| 197 |
json_file.write(data_to_save_encode)
|
| 198 |
|
| 199 |
excel_file_path = os.path.join(flagging_dir, f"{base_filename}.xlsx")
|
| 200 |
+
try:
|
| 201 |
+
save_to_excel(json_output, excel_file_path, image_file_path)
|
| 202 |
+
except Exception as e:
|
| 203 |
+
print(f"Error while saving to excel: {e}")
|
| 204 |
|
| 205 |
# Upload files to Google Drive
|
| 206 |
+
google_drive_client_current = GoogleDriveClient(json_key_path='secrets/GOOGLE_SERVICE_ACCOUNT_KEY.json')
|
| 207 |
if google_drive_client_current:
|
| 208 |
try:
|
| 209 |
+
image_folder_id = '10qtum6ykbGTyu7vvw59i3h1XSY3-lRpo'
|
| 210 |
image_link = google_drive_client_current.upload_file(image_save_path, image_folder_id)
|
| 211 |
json_link = google_drive_client_current.upload_file(json_file_path, image_folder_id)
|
| 212 |
excel_link = google_drive_client_current.upload_file(excel_file_path, image_folder_id)
|
|
|
|
| 220 |
|
| 221 |
except Exception as e:
|
| 222 |
print(f"Error while saving flag data: {e}")
|
| 223 |
+
links = f"Image: {image_link}\nJSON: {json_link}\nExcel: {excel_link} \n shared lofder: https://drive.google.com/drive/folders/10qtum6ykbGTyu7vvw59i3h1XSY3-lRpo?usp=drive_link \n"
|
| 224 |
return save_button_update, save_button_update, save_button_update, links
|
| 225 |
|
| 226 |
+
|
| 227 |
def update_prompt_from_radio(prompt_name):
|
| 228 |
if prompt_name == "prompt_v1":
|
| 229 |
+
return read_prompt_from_file("common/prompt_v1.txt")
|
| 230 |
elif prompt_name == "prompt_v2":
|
| 231 |
+
return read_prompt_from_file("common/prompt_v2.txt")
|
| 232 |
elif prompt_name == "prompt_v3":
|
| 233 |
+
return read_prompt_from_file("common/prompt_v3.txt")
|
| 234 |
else:
|
| 235 |
+
return read_prompt_from_file("common/prompt_v1.txt")
|
| 236 |
|
| 237 |
+
|
| 238 |
+
google_drive_client = GoogleDriveClient(json_key_path='secrets/GOOGLE_SERVICE_DRIVE_KEY_435817.json')
|
| 239 |
+
vertex_ai_client = VertexAIService(json_key_path='secrets/GOOGLE_VERTEX_AI_KEY_435817.json')
|
| 240 |
|
| 241 |
key = None
|
| 242 |
+
key_file_path = 'secrets/OPENAI_AI_KEY.txt'
|
| 243 |
if os.path.exists(key_file_path):
|
| 244 |
try:
|
| 245 |
with open(key_file_path, 'r') as key_file:
|
|
|
|
| 250 |
open_ai_client = OpenAIService(api_key=key)
|
| 251 |
|
| 252 |
with gr.Blocks() as iface:
|
| 253 |
+
gr.Markdown("# ReceiptAI")
|
| 254 |
+
gr.Markdown("ReceiptAI")
|
| 255 |
|
| 256 |
with gr.Row():
|
| 257 |
with gr.Column(scale=1):
|
| 258 |
image_input = gr.Image(type="filepath")
|
| 259 |
+
model_radio = gr.Radio(model_names, label="Choose model/QR-processing(Slovakia)", value=model_names[0])
|
| 260 |
+
prompt_radio = gr.Radio(prompt_names, label="Choose prompt", value=prompt_names[0], visible=isFullVersion)
|
| 261 |
+
temperature_slider = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label="Temperatura", value=0.0,
|
| 262 |
+
visible=isFullVersion)
|
| 263 |
+
system_instruction = gr.Textbox(label="System Instruction", visible=isFullVersion, value=system_instruction)
|
| 264 |
+
custom_prompt = gr.Textbox(label="prompt text", visible=isFullVersion, value=prompt_default)
|
| 265 |
with gr.Row():
|
| 266 |
+
submit_button = gr.Button("Receipt recognizing ")
|
| 267 |
|
| 268 |
with gr.Column(scale=2):
|
| 269 |
+
model_output = gr.Textbox(label="MODEL/QR-processing(Slovakia)", lines=1, interactive=isFullVersion)
|
| 270 |
json_output = gr.Textbox(label="Result as json")
|
| 271 |
store_info_output = gr.Textbox(label="Store Information", lines=4)
|
| 272 |
items_list = gr.Dataframe(
|
| 273 |
+
headers=["Item Name", "Category", "Unit Price", "Quantity", "Unit", "Total Price", "Discount",
|
| 274 |
+
"Item price with tax", "Grand Total"],
|
| 275 |
label="Items List")
|
| 276 |
+
taxes_list = gr.Dataframe(
|
| 277 |
+
headers=["Tax Name", "%", "tax from amount", "tax", "total", "tax included"],
|
| 278 |
+
label="Tax List")
|
| 279 |
comments_output = gr.Textbox(label="Comments", visible=True, lines=4, interactive=True)
|
| 280 |
with gr.Row():
|
| 281 |
save_good_button = gr.Button(value="Save as Good", interactive=False)
|
| 282 |
+
save_average_button = gr.Button(value="Save as Average", interactive=False)
|
| 283 |
save_poor_button = gr.Button(value="Save as Poor", interactive=False)
|
| 284 |
file_links_output = gr.Textbox(label="File Links", interactive=False, visible=True)
|
| 285 |
submit_button.click(fn=process_image,
|
| 286 |
inputs=[image_input, model_radio, prompt_radio, temperature_slider, system_instruction,
|
| 287 |
custom_prompt],
|
| 288 |
+
outputs=[model_output, json_output, store_info_output, items_list, taxes_list, comments_output,
|
| 289 |
+
save_good_button, save_average_button, save_poor_button, file_links_output])
|
| 290 |
common_inputs = [image_input, model_radio, prompt_radio, temperature_slider, custom_prompt, model_output,
|
| 291 |
json_output, store_info_output, items_list, comments_output, system_instruction]
|
| 292 |
|
|
|
|
| 305 |
)
|
| 306 |
return save_good_update, save_avg_update, save_poor_update, file_links
|
| 307 |
|
| 308 |
+
|
| 309 |
# Use the same common_inputs for all buttons but ensure the correct values are passed
|
| 310 |
save_good_button.click(
|
| 311 |
fn=lambda *args: save_flag_data_wrapper("Good", *args),
|
|
|
|
| 325 |
outputs=[save_good_button, save_average_button, save_poor_button, file_links_output]
|
| 326 |
)
|
| 327 |
prompt_radio.change(fn=update_prompt_from_radio, inputs=[prompt_radio], outputs=[custom_prompt])
|
| 328 |
+
gr.Examples(examples=example_list_sl,
|
| 329 |
inputs=[image_input, model_radio, prompt_radio, temperature_slider, custom_prompt],
|
| 330 |
+
label="Examples for Slovakia")
|
| 331 |
+
if isFullVersion:
|
| 332 |
+
gr.Examples(examples=example_list_ua,
|
| 333 |
+
inputs=[image_input, model_radio, prompt_radio, temperature_slider, custom_prompt],
|
| 334 |
+
label="Examples for Ukrainian")
|
| 335 |
|
| 336 |
+
gr.Examples(examples=example_list_us,
|
| 337 |
+
inputs=[image_input, model_radio, prompt_radio, temperature_slider, custom_prompt],
|
| 338 |
+
label="Examples for US")
|
| 339 |
|
| 340 |
+
gr.Examples(examples=example_list_canada,
|
| 341 |
+
inputs=[image_input, model_radio, prompt_radio, temperature_slider, custom_prompt],
|
| 342 |
+
label="Examples for Canada")
|
| 343 |
|
| 344 |
+
gr.Examples(examples=example_france,
|
| 345 |
+
inputs=[image_input, model_radio, prompt_radio, temperature_slider, custom_prompt],
|
| 346 |
+
label="Examples for France")
|
| 347 |
|
| 348 |
+
iface.launch(server_name="0.0.0.0", server_port=7860)
|
common/enum/__init__.py
ADDED
|
File without changes
|
common/enum/ai_service_error.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from enum import Enum
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class AiServiceError(Enum):
|
| 5 |
+
RETRY_FETCH = "RETRY_FETCH"
|
| 6 |
+
RETAKE_PHOTO = "RETAKE_PHOTO"
|
common/enum/payment_method.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from enum import Enum
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class PaymentMethod(Enum):
|
| 5 |
+
CARD = 'CARD'
|
| 6 |
+
CASH = 'CASH'
|
| 7 |
+
UNKNOWN = 'UNKNOWN'
|
common/exceptions.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from api_exception.exception import APIException
|
| 2 |
+
|
| 3 |
+
from common.enum.ai_service_error import AiServiceError
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class Conflict(APIException):
|
| 7 |
+
status_code = 409
|
| 8 |
+
default_detail = "Conflict"
|
| 9 |
+
default_code = "conflict"
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class AiServiceException(APIException):
|
| 13 |
+
status_code = 400
|
| 14 |
+
|
| 15 |
+
def __init__(self, ai_error_code: AiServiceError, detail=None, status_code=None):
|
| 16 |
+
self.error_code = ai_error_code.value
|
| 17 |
+
self.detail = detail if detail is not None else ai_error_code.value
|
| 18 |
+
if status_code is not None:
|
| 19 |
+
self.status_code = status_code
|
| 20 |
+
super().__init__(detail=self.detail)
|
prompt_v1.txt → common/prompt_v1.txt
RENAMED
|
@@ -6,38 +6,45 @@ Output Format: Return the output as a JSON object with the following structure:
|
|
| 6 |
|
| 7 |
{
|
| 8 |
"store_name": string, -- Exact name of the store as found on the receipt. It`s not always the bigger text. Find the correct name of the shop/restaurant
|
| 9 |
-
"
|
| 10 |
-
"
|
|
|
|
|
|
|
| 11 |
"currency": string, -- Currency code (e.g., "EUR", "USD", "UAH") based on the detected currency symbol. Don`t put here currency symbol, only code.
|
| 12 |
-
"
|
| 13 |
-
"
|
| 14 |
-
"
|
| 15 |
-
"
|
| 16 |
-
"
|
| 17 |
-
"
|
|
|
|
|
|
|
|
|
|
| 18 |
"items": [
|
| 19 |
{
|
| 20 |
"name": string, -- Full item name (even if it spans multiple lines)
|
| 21 |
-
"
|
| 22 |
-
"
|
| 23 |
-
"
|
| 24 |
-
"
|
| 25 |
-
"
|
| 26 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
}
|
| 28 |
]
|
| 29 |
}
|
| 30 |
|
| 31 |
-
#Handling Item Price, Quantity, and Name Layout:
|
| 32 |
-
When processing items, note that the price, unit price, and quantity may appear above the item name. For example, if the receipt shows:
|
| 33 |
-
5 * 23.00 = 115.0
|
| 34 |
-
Milk
|
| 35 |
-
Interpret this as:
|
| 36 |
-
Quantity: 5
|
| 37 |
-
Unit Price: 23.00
|
| 38 |
-
Item Name: Milk
|
| 39 |
-
Total Price: 115.0
|
| 40 |
-
|
| 41 |
#Additional Notes:
|
| 42 |
1. If no receipt is detected: Return "Receipt not found."
|
| 43 |
2. Handle various languages (including non-Latin scripts) and keep text in the original script unless translation is explicitly required.
|
|
@@ -46,6 +53,8 @@ Total Price: 115.0
|
|
| 46 |
5. Some receipts could be, for example, from McDonald`s restaurant, where in receipts under menu name could be written components of this menu. In this case you should extract only menu name.
|
| 47 |
6. The total amount may not always be the largest number; ensure the context is understood from surrounding text.
|
| 48 |
7. Tips and Charity Donations: Extract and sum tips and charity donations, storing the total under the tips field.
|
| 49 |
-
8. Convert
|
| 50 |
9. Handle ambiguous data consistently. If there's ambiguity about price, quantity, or any other information, make the best effort to extract it, or return "unknown."
|
| 51 |
-
|
|
|
|
|
|
|
|
|
| 6 |
|
| 7 |
{
|
| 8 |
"store_name": string, -- Exact name of the store as found on the receipt. It`s not always the bigger text. Find the correct name of the shop/restaurant
|
| 9 |
+
"country": string, -- Define country if available; otherwise, "unknown". Identify country by details on the receipt. Use receipt address or language if explicit country info is lacking.
|
| 10 |
+
"receipt_type": string, -- Define receipt type (e.g. Restaurant/Shop/Other) if available; otherwise, "unknown"
|
| 11 |
+
"address": string, -- Full address, if available; otherwise, "unknown"
|
| 12 |
+
"datetime": "YYYY.MM.DD HH:MM:SS", -- Convert all date formats to this standard
|
| 13 |
"currency": string, -- Currency code (e.g., "EUR", "USD", "UAH") based on the detected currency symbol. Don`t put here currency symbol, only code.
|
| 14 |
+
"sub_total_amount": 0.00, -- This represents the total cost of all items and services on the receipt before any tips, or additional charges are applied. If sub_total_amount is not present on the receipt, set "unknown"
|
| 15 |
+
"total_price": 0.00, -- The final total amount from the receipt (in the majority of situations this one is bigger then other values + it could be as bold font). The total amount may not always be the largest number; ensure the context is understood from surrounding text.
|
| 16 |
+
"total_discount": 0.00, -- Total discount applied based on individual item discounts or explicit discount information
|
| 17 |
+
"all_items_price_with_tax": True/False -- Indicates whether taxes are included in the prices of items. Set to True if taxes are included, False if they are not included. If it cannot be determined, set to "unknown".
|
| 18 |
+
"payment_method": "card", "cash", or "unknown", -- Detect payment method based on keywords like "card", "cash", "master card", "visa", e.t. or if missing, use "unknown"
|
| 19 |
+
"rounding": 0.00, -- If rounding is not specified on the receipt, use 0.0
|
| 20 |
+
"tax": 0.00, -- If tax is not found or mentioned, use 0.0
|
| 21 |
+
"taxes_not_included_sum": 0.0 -- Represents the total amount of taxes that are not included in the final total on the receipt. This is applicable in situations where taxes are itemized separately, such as in the United States. If there are no separate taxes, set to 0.0.
|
| 22 |
+
"tips": 0.00, -- If tips is not found or mentioned, use 0.0
|
| 23 |
"items": [
|
| 24 |
{
|
| 25 |
"name": string, -- Full item name (even if it spans multiple lines)
|
| 26 |
+
"quantity": 0.000, -- Quantity of the item, default 1.0 if it wasn`t written
|
| 27 |
+
"measurement_unit": string, -- Use the format "ks", "kg", etc. If not specified, default to "ks"
|
| 28 |
+
"total_price_without_discount": 0.00, -- price without any discount for a single item. Always extract this value directly from the receipt
|
| 29 |
+
"unit_price": 0.00, -- Price per unit without any discount, if available. If not, write here the same value as for total_price_without_discount. Can be negative
|
| 30 |
+
"total_price_with_discount": 0.00 - -- This is the full price for a single item after considering all applicable discounts.
|
| 31 |
+
"discount": 0.00, -- If discount isn't listed, assume 0.00
|
| 32 |
+
"category": string -- Category choose fromlist:Food,Beverages,Personal Care, Beauty & Health,Household Items,Electronics & Appliances,Clothing & Accessories,Home & Furniture,Entertainment & Media,Sports & Outdoors,Car,Baby Products,Stationery,Pet Supplies,Health & Fitness Services,Travel & Transportation,Insurance & Financial Services,Utilities,Gifts & Specialty Items,Services,Other options
|
| 33 |
+
"item_price_with_tax": string -- "True"/"False". Indicating whether the item prices include tax.
|
| 34 |
+
}
|
| 35 |
+
]
|
| 36 |
+
"taxs_items": [
|
| 37 |
+
{
|
| 38 |
+
"tax_name": string -- The name of the tax or tax rate.
|
| 39 |
+
"percentage": 0.00 --The tax percentage.
|
| 40 |
+
"tax_from_amount": 0.00 -- The amount before tax.
|
| 41 |
+
"tax": 0.00 -- The tax amount itself.
|
| 42 |
+
"total": 0.00 -- The total amount including tax.
|
| 43 |
+
"tax_included": string -- "True"/"False" indicating whether taxes are included in the item prices. Set to True if there is no separate line for tax on the receipt, or if it explicitly states that taxes are included. Otherwise, set to False
|
| 44 |
}
|
| 45 |
]
|
| 46 |
}
|
| 47 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
#Additional Notes:
|
| 49 |
1. If no receipt is detected: Return "Receipt not found."
|
| 50 |
2. Handle various languages (including non-Latin scripts) and keep text in the original script unless translation is explicitly required.
|
|
|
|
| 53 |
5. Some receipts could be, for example, from McDonald`s restaurant, where in receipts under menu name could be written components of this menu. In this case you should extract only menu name.
|
| 54 |
6. The total amount may not always be the largest number; ensure the context is understood from surrounding text.
|
| 55 |
7. Tips and Charity Donations: Extract and sum tips and charity donations, storing the total under the tips field.
|
| 56 |
+
8. Convert datetime to the "YYYY.MM.DD HH:MM:SS" format, regardless of how they appear on the receipt (e.g., MM/DD/YY, DD-MM-YYYY).
|
| 57 |
9. Handle ambiguous data consistently. If there's ambiguity about price, quantity, or any other information, make the best effort to extract it, or return "unknown."
|
| 58 |
+
10. Be flexible in handling varied receipt layouts, item name formats, and currencies.
|
| 59 |
+
11. The unit_price/price/total_price/total_price_without_discount for an item can be negative
|
| 60 |
+
12. After the total amount may be information about taxes, in separate tax items. Define them in taxs_items
|
prompt_v2.txt → common/prompt_v2.txt
RENAMED
|
File without changes
|
common/prompt_v3.txt
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#Your Task: Receipt Recognition and Data Extraction
|
| 2 |
+
|
| 3 |
+
You are tasked with extracting structured information from receipts. Receipts will come from various countries, in different languages, and can have different layouts and formats. Your goal is to parse the receipt text, identify the receipt type (store, cafe/restaurant, or payment for services), and return the data in JSON format with the required fields. Follow the specific instructions below to ensure accurate extraction.
|
| 4 |
+
|
| 5 |
+
#Required Fields:
|
| 6 |
+
|
| 7 |
+
1. Receipt Type: Identify the type of receipt. It could be from:
|
| 8 |
+
* Store: Typically involves grocery or retail items.
|
| 9 |
+
* Cafe/Restaurant: Typically involves food and beverage items, table numbers, or tipping sections.
|
| 10 |
+
* Payment for Services: This type of receipt may involve service fees or professional services.
|
| 11 |
+
2. Receipt Number: Extract the unique receipt number, typically found at the top of the receipt.
|
| 12 |
+
3. Store/Business Name: Extract the name of the store, cafe, restaurant, or service provider.
|
| 13 |
+
4. Store Address: Extract the address of the store, including city and country if available.
|
| 14 |
+
5. Date: Extract the date of the transaction and format it as YYYY-MM-DD HH:MM.
|
| 15 |
+
6. Currency: Extract the currency if explicitly mentioned (e.g., EUR, USD). If the currency is not specified, detect the language of the receipt and infer the currency based on the country where the language is predominantly used. For example, if the receipt is in Ukrainian, set the currency to UAH (Ukrainian Hryvnia).
|
| 16 |
+
7. Payment Method: Identify whether the payment was made by "card" or "cash."
|
| 17 |
+
8. Total Amount: Extract the total amount of the transaction. This is typically located at the end of the receipt, often highlighted in bold or a larger font.
|
| 18 |
+
9. Total Discount: Extract the total discount if explicitly mentioned. If not, calculate the total discount by summing up the discounts for individual items.
|
| 19 |
+
10. Tax: Extract the total tax amount if it is listed on the receipt.
|
| 20 |
+
|
| 21 |
+
#Item-Level Details:
|
| 22 |
+
|
| 23 |
+
For each item on the receipt, extract the following details:
|
| 24 |
+
|
| 25 |
+
1. Item Name: Extract the full name of each item. Some items may have names split across multiple lines; in this case, concatenate the lines until you encounter a quantity or unit of measurement (e.g., "2ks"), which marks the end of the item name or on the next line. You should extract full name till statements as, for example, "1 ks" or "1 ks * 2"
|
| 26 |
+
2. Unit Price: Extract the price per unit for each item.
|
| 27 |
+
3. Quantity: Extract the quantity of each item, including the unit of measurement (e.g., "ks" for pieces, "kg" for kilograms).
|
| 28 |
+
4. Price: Extract the final price for each item.
|
| 29 |
+
5. Discount: Extract any discount applied to the item. If no discount is provided, set it to 0.
|
| 30 |
+
6. Category: Automatically assign a category based on the item name. For groceries, assign relevant subcategories such as Dairy, Bakery, Fruits, etc. If this receipt was from restaurant - you should put category only from this list: Food, Drinks.
|
| 31 |
+
|
| 32 |
+
#Special Cases:
|
| 33 |
+
|
| 34 |
+
1. Cafe/Restaurant Receipts: If the receipt is from a cafe or restaurant, handle additional fields like:
|
| 35 |
+
* Table Number: Extract the table number if available, often printed near the top of the receipt.
|
| 36 |
+
* Tips: Extract any tip amounts explicitly listed or infer from the total paid amount minus the original bill amount.
|
| 37 |
+
* Service Charges: Some restaurants may include an automatic service charge, which should be listed separately from the tax or tips.
|
| 38 |
+
* Order Type: Identify whether the order was "dine-in" or "takeaway."
|
| 39 |
+
2. Missing Currency: If no currency is mentioned on the receipt, infer the local currency by detecting the language and country of origin. For example, a receipt in French would use EUR, while one in Ukrainian would use UAH.
|
| 40 |
+
3. Multi-line Item Names: If an item name spans multiple lines, merge the lines to form the complete name. Stop merging when a quantity or unit of measurement is encountered.
|
| 41 |
+
4. Total Amount: The total amount is often larger than other numbers or displayed in bold at the bottom of the receipt. Make sure to capture this accurately.
|
| 42 |
+
5. Total Discount: If no total discount is listed, sum the discounts for each individual item.
|
| 43 |
+
6. Rounding Adjustments: Some receipts may include a "rounding" line item, where the total amount is adjusted (typically for cash payments) to avoid dealing with fractions of currency (e.g., rounding to the nearest 0.05 in some countries). If a rounding adjustment is present, extract the value of the rounding adjustment and reflect it in the total amount. For example:
|
| 44 |
+
* Total Before Rounding: 19.97
|
| 45 |
+
* Rounding: -0.02
|
| 46 |
+
* Final Total: 19.95 If the rounding adjustment is found, include it as a separate field in the JSON output under "rounding_adjustment", and ensure that the "total_amount" reflects the final adjusted total.
|
| 47 |
+
7. Taxes: Receipts can handle taxes in various ways, and the system should be prepared to capture these scenarios:
|
| 48 |
+
* Tax-Inclusive Pricing: In some countries or for certain receipts, taxes are already included in the item price and not listed separately. If the receipt mentions that taxes are included in prices, record the "tax" field as 0 and note that taxes are included in the item prices.
|
| 49 |
+
* Multiple Tax Rates: Some receipts may include multiple tax rates (e.g., different VAT rates for different items). In this case, extract each tax rate and the corresponding tax amounts, and store them in a separate list of tax breakdowns. For example, the receipt might show "5% VAT" and "15% VAT" for different categories of goods:
|
| 50 |
+
** "taxes": [{"rate": "5%", "amount": 1.00}, {"rate": "15%", "amount": 3.50}]
|
| 51 |
+
* Missing Tax Information: In some cases, the receipt might not clearly mention taxes, but you may infer them based on standard rates in the country of origin. If no explicit tax amount is listed and you are unable to infer it, set the tax to "unknown" or null in the JSON output.
|
| 52 |
+
* Tax-Exempt Items: Some items on the receipt may be tax-exempt. If this is indicated, ensure that these items are excluded from any tax calculations. Note these in the item-level details with "tax_exempt": true and make sure the "tax" field reflects the correct amount for taxable items only.
|
| 53 |
+
* Service Charges vs. Taxes: Sometimes service charges may be listed separately from taxes (common in restaurants). Ensure that service charges are not included in the tax amount, and store them under the "service_charge" field.
|
| 54 |
+
* Tax Breakdown and Total: If both individual item taxes and a total tax amount are listed, the system should ensure consistency between the sum of item-level taxes and the total tax listed at the bottom of the receipt.
|
| 55 |
+
8. In certain receipt formats, the quantity and unit price may appear before the item name. When processing such receipts, the goal is to correctly extract the quantity, unit price, and item name in their proper order. For example, if one line of the receipt shows "5 * 23.00 = 115.0" and the next line displays "Milk," the system should interpret this as:
|
| 56 |
+
* Quantity: 5 units
|
| 57 |
+
* Unit Price: 23.00
|
| 58 |
+
* Item Name: Milk
|
| 59 |
+
* Total Price: 115.0 This approach should be applied consistently throughout the entire receipt to extract data accurately.
|
| 60 |
+
|
| 61 |
+
#JSON Output Format:
|
| 62 |
+
|
| 63 |
+
{
|
| 64 |
+
"receipt_type": "string",
|
| 65 |
+
"receipt_number": "string",
|
| 66 |
+
"store_name": "string",
|
| 67 |
+
"store_address": "string",
|
| 68 |
+
"date_time": "string",
|
| 69 |
+
"currency": "string",
|
| 70 |
+
"payment_method": "string",
|
| 71 |
+
"total_amount": "number",
|
| 72 |
+
"total_discount": "number",
|
| 73 |
+
"tax": "number",
|
| 74 |
+
"taxes": [
|
| 75 |
+
{
|
| 76 |
+
"rate": "string",
|
| 77 |
+
"amount": "number"
|
| 78 |
+
}
|
| 79 |
+
],
|
| 80 |
+
"rounding_adjustment": "number",
|
| 81 |
+
"rounded_total_aount": "number",
|
| 82 |
+
"items": [
|
| 83 |
+
{
|
| 84 |
+
"name": "string",
|
| 85 |
+
"unit_price": "number",
|
| 86 |
+
"quantity": {
|
| 87 |
+
"amount": "number",
|
| 88 |
+
"unit_of_measurement": "string"
|
| 89 |
+
},
|
| 90 |
+
"price": "number",
|
| 91 |
+
"discount": "number",
|
| 92 |
+
"category": "string",
|
| 93 |
+
"tax_exempt": "boolean"
|
| 94 |
+
}
|
| 95 |
+
],
|
| 96 |
+
"cafe_additional_info": {
|
| 97 |
+
"table_number": "string",
|
| 98 |
+
"tips": "number",
|
| 99 |
+
"service_charge": "number",
|
| 100 |
+
"order_type": "string"
|
| 101 |
+
}
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
#Additional Notes:
|
| 105 |
+
|
| 106 |
+
1. You should handle receipts in various languages and from different countries.
|
| 107 |
+
2. Pay special attention to formatting differences and edge cases, such as multi-line item names, missing currency symbols, or cafe/restaurant-specific information.
|
| 108 |
+
3. Always ensure the output is well-structured and follows the JSON format provided.
|
| 109 |
+
4. The "rounding_adjustment" field should reflect the value by which the total was adjusted due to rounding. If no rounding adjustment is present, it can be set to 0 or omitted from the output.
|
| 110 |
+
5. Ensure that the final "total_amount" field reflects the total after any rounding adjustment has been applied.
|
| 111 |
+
6. Inclusive Taxes: If taxes are included in the item prices, set the "tax" field to 0 and adjust the item prices accordingly.
|
| 112 |
+
7. Multiple Tax Rates: The "taxes" field provides a detailed breakdown for receipts with different tax rates. This field is optional and can be excluded if only a single tax amount is listed.
|
| 113 |
+
8. Tax-Exempt Items: Mark tax-exempt items with the "tax_exempt": true field.
|
| 114 |
+
9. Service Charges vs. Taxes: Ensure that service charges are captured separately from taxes in the "service_charge" field.
|
| 115 |
+
10. Return the full JSON object with all available information. If any information is unclear or missing, include it as "unknown" or "not available" in the output.
|
| 116 |
+
11. Your final response should be in valid JSON format with no additional text.
|
common/utils/__init__.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from os import environ
|
| 2 |
+
|
| 3 |
+
import requests
|
| 4 |
+
|
| 5 |
+
from .utils import *
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def get_env_var(key):
|
| 9 |
+
try:
|
| 10 |
+
return environ[key]
|
| 11 |
+
except KeyError:
|
| 12 |
+
logger.debug(f"Missing {key} environment variable.")
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def get_user_info(request):
|
| 16 |
+
access_token = request.META.get('HTTP_AUTHORIZATION', '').split(' ')[1]
|
| 17 |
+
userinfo_response = requests.get(
|
| 18 |
+
'https://dev-jy1enj724oy2las3.us.auth0.com/userinfo',
|
| 19 |
+
headers={'Authorization': f'Bearer {access_token}'}
|
| 20 |
+
)
|
| 21 |
+
if userinfo_response.status_code == 200:
|
| 22 |
+
userinfo = userinfo_response.json()
|
| 23 |
+
return userinfo
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def get_model_prices(model_name: str):
|
| 27 |
+
"""
|
| 28 |
+
Returns input and output prices for models for 1k tokens
|
| 29 |
+
|
| 30 |
+
:param model_name: str["gpt-4o" | "gpt-4o-mini" | "gemini-1.5-pro" | "gemini-1.5-flash"]
|
| 31 |
+
:return: Tuple[input_price, output_price]
|
| 32 |
+
"""
|
| 33 |
+
env_model_name = model_name.upper().replace("-", "_")
|
| 34 |
+
|
| 35 |
+
input_price = get_env_var(f'{env_model_name}_INPUT_PRICE_1K')
|
| 36 |
+
output_price = get_env_var(f'{env_model_name}_OUTPUT_PRICE_1K')
|
| 37 |
+
|
| 38 |
+
return float(output_price), float(input_price)
|
common/utils/utils.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
import time
|
| 3 |
+
from io import BytesIO
|
| 4 |
+
|
| 5 |
+
from PIL import Image
|
| 6 |
+
from loguru import logger
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def crop_image(img: Image, new_size: tuple[int, int]) -> Image:
|
| 10 |
+
"""
|
| 11 |
+
Crop an image to the specified size, maintaining the aspect ratio.
|
| 12 |
+
:param img: The image to crop.
|
| 13 |
+
:param new_size: The target size of the cropped image.
|
| 14 |
+
:return: The cropped image.
|
| 15 |
+
"""
|
| 16 |
+
width, height = img.size
|
| 17 |
+
|
| 18 |
+
target_width, target_height = new_size
|
| 19 |
+
|
| 20 |
+
aspect_ratio = width / height
|
| 21 |
+
target_aspect_ratio = target_width / target_height
|
| 22 |
+
|
| 23 |
+
if aspect_ratio > target_aspect_ratio:
|
| 24 |
+
new_height = target_height
|
| 25 |
+
new_width = int(aspect_ratio * new_height)
|
| 26 |
+
else:
|
| 27 |
+
new_width = target_width
|
| 28 |
+
new_height = int(new_width / aspect_ratio)
|
| 29 |
+
|
| 30 |
+
img = img.resize((new_width, new_height), Image.Resampling.LANCZOS)
|
| 31 |
+
|
| 32 |
+
left = (new_width - target_width) / 2
|
| 33 |
+
top = (new_height - target_height) / 2
|
| 34 |
+
right = (new_width + target_width) / 2
|
| 35 |
+
bottom = (new_height + target_height) / 2
|
| 36 |
+
|
| 37 |
+
return img.crop((left, top, right, bottom))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def encode_image_to_webp_base64(filepath):
|
| 41 |
+
"""
|
| 42 |
+
Encodes an image file to WEBP and then to a Base64 string.
|
| 43 |
+
|
| 44 |
+
:param filepath: Path to the image file
|
| 45 |
+
:return: Base64 encoded string of the WEBP image
|
| 46 |
+
:raises ValueError: If filepath is None or other issues occur during encoding
|
| 47 |
+
"""
|
| 48 |
+
start_time = time.time()
|
| 49 |
+
if filepath is None:
|
| 50 |
+
raise ValueError("File path is None.")
|
| 51 |
+
|
| 52 |
+
try:
|
| 53 |
+
pil_image = Image.open(filepath)
|
| 54 |
+
|
| 55 |
+
if pil_image.mode == 'RGBA':
|
| 56 |
+
pil_image = pil_image.convert('RGB')
|
| 57 |
+
|
| 58 |
+
buffered = BytesIO()
|
| 59 |
+
pil_image.save(buffered, format="WEBP")
|
| 60 |
+
|
| 61 |
+
base64_image = base64.b64encode(buffered.getvalue()).decode('utf-8')
|
| 62 |
+
return base64_image
|
| 63 |
+
|
| 64 |
+
except Exception as e:
|
| 65 |
+
raise ValueError(f"Failed to encode image to WEBP base64: {str(e)}")
|
| 66 |
+
finally:
|
| 67 |
+
end_time = time.time()
|
| 68 |
+
elapsed_time = end_time - start_time
|
| 69 |
+
logger.info(f"Encoding image to WebP and Base64 spent {elapsed_time:.2f} seconds to process.")
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def encode_webp_to_base64(image_file):
|
| 73 |
+
"""
|
| 74 |
+
Encodes a WEBP image file (from HTTP upload) directly to a Base64 string.
|
| 75 |
+
|
| 76 |
+
:param image_file: InMemoryUploadedFile object containing the WEBP image
|
| 77 |
+
:return: Base64 encoded string of the WEBP image
|
| 78 |
+
:raises ValueError: If image_file is None or other issues occur during encoding
|
| 79 |
+
"""
|
| 80 |
+
start_time = time.time()
|
| 81 |
+
if image_file is None:
|
| 82 |
+
raise ValueError("Image file is None.")
|
| 83 |
+
|
| 84 |
+
try:
|
| 85 |
+
# Ensure that the file is read as binary data
|
| 86 |
+
base64_image = base64.b64encode(image_file.read()).decode('utf-8')
|
| 87 |
+
return base64_image
|
| 88 |
+
|
| 89 |
+
except Exception as e:
|
| 90 |
+
raise ValueError(f"Failed to encode WEBP image to Base64: {str(e)}")
|
| 91 |
+
finally:
|
| 92 |
+
end_time = time.time()
|
| 93 |
+
elapsed_time = end_time - start_time
|
| 94 |
+
logger.info(f"Encoding image to Base64 spent {elapsed_time:.2f} seconds to process.")
|
convert_to_webp.py
CHANGED
|
@@ -26,5 +26,5 @@ def process_images_in_directory(directory):
|
|
| 26 |
if __name__ == "__main__":
|
| 27 |
current_directory = os.getcwd()
|
| 28 |
print(f"current_directory: {current_directory}")
|
| 29 |
-
directory = "./
|
| 30 |
process_images_in_directory(directory)
|
|
|
|
| 26 |
if __name__ == "__main__":
|
| 27 |
current_directory = os.getcwd()
|
| 28 |
print(f"current_directory: {current_directory}")
|
| 29 |
+
directory = "./examples/"
|
| 30 |
process_images_in_directory(directory)
|
examples/5404832557079585491.webp
DELETED
Git LFS Details
|
examples/ATB_11.webp
DELETED
Git LFS Details
|
examples/Sportisimo.webp
DELETED
Git LFS Details
|
examples/cafe.webp
DELETED
Git LFS Details
|
examples/farmacy_1.webp
DELETED
Git LFS Details
|
examples/farmacy_2.webp
DELETED
Git LFS Details
|
examples/fatlouis.webp
DELETED
Git LFS Details
|
examples/garm_3.webp
DELETED
Git LFS Details
|
examples/image_2024_09_25T12_13_01_811Z.webp
DELETED
Git LFS Details
|
examples/lidl1.webp
DELETED
Git LFS Details
|
examples/lidl2.webp
DELETED
Git LFS Details
|
examples/photo_2024-09-10_10-06-14.webp
DELETED
Git LFS Details
|
examples/photo_2024-09-10_10-06-24.webp
DELETED
Git LFS Details
|
examples/photo_2024-09-10_10-06-28.webp
DELETED
Git LFS Details
|
examples/photo_2024-09-24_14-50-34.webp
DELETED
Git LFS Details
|
examples/tiket.webp
DELETED
Git LFS Details
|
examples_canada/photo_2024-10-09_14-21-54.webp
DELETED
|
Binary file (51.1 kB)
|
|
|
examples_canada/photo_2024-10-09_14-23-03.webp
DELETED
|
Binary file (140 kB)
|
|
|
examples_canada/photo_2024-10-10_15-23-22.webp
DELETED
|
Binary file (37.1 kB)
|
|
|
examples_france/photo_1_2024-10-02_00-08-53.webp
DELETED
|
Binary file (88.7 kB)
|
|
|
examples_france/photo_2024-10-07_21-46-05.webp
DELETED
|
Binary file (117 kB)
|
|
|
examples_france/photo_2024-10-07_21-46-27.webp
DELETED
|
Binary file (119 kB)
|
|
|
examples_france/photo_2_2024-10-02_00-08-53.webp
DELETED
|
Binary file (80.5 kB)
|
|
|
examples_france/photo_3_2024-10-02_00-08-53.webp
DELETED
|
Binary file (93.8 kB)
|
|
|
examples_france/photo_4_2024-10-02_00-08-53.webp
DELETED
|
Binary file (78.3 kB)
|
|
|
examples_france/photo_5_2024-10-02_00-08-53.webp
DELETED
|
Binary file (62.5 kB)
|
|
|
examples_france/photo_6_2024-10-02_00-08-53.webp
DELETED
|
Binary file (92.9 kB)
|
|
|
examples_france/photo_7_2024-10-02_00-08-53.webp
DELETED
|
Binary file (81.9 kB)
|
|
|
examples_france/photo_8_2024-10-02_00-08-53.webp
DELETED
|
Binary file (86.9 kB)
|
|
|
examples_us/Photo Sep 29 2024, 15 43 17.webp
DELETED
|
Binary file (670 kB)
|
|
|
examples_us/Photo Sep 29 2024, 15 43 30.webp
DELETED
|
Binary file (510 kB)
|
|
|
examples_us/Photo Sep 29 2024, 15 43 48.webp
DELETED
|
Binary file (827 kB)
|
|
|
examples_us/Photo Sep 29 2024, 15 44 21.webp
DELETED
|
Binary file (797 kB)
|
|
|
examples_us/Photo Sep 29 2024, 15 44 42.webp
DELETED
|
Binary file (686 kB)
|
|
|
examples_us/Photo Sep 29 2024, 15 45 13.webp
DELETED
|
Binary file (518 kB)
|
|
|
examples_us/Photo Sep 29 2024, 15 45 26.webp
DELETED
|
Binary file (583 kB)
|
|
|