add LLM summarisation

app.py

@@ -7,10 +7,20 @@ from flask import Flask, jsonify, render_template, request, send_file
 import base64
 import numpy as np
 import easyocr
+from huggingface_hub import InferenceClient
+
 app = Flask(__name__)
 
 reader = easyocr.Reader(['en']) 
 
+HF_API_KEY=os.getenv("HF_API_KEY")
+
+client = InferenceClient(api_key=HF_API_KEY)
+
+
+
+
+
 @app.route("/")
 def index():
     return render_template("index.html")
@@ -19,6 +29,7 @@ def index():
 @app.route('/analyse', methods=['POST'])
 def analyse():
     frame = request.json.get('key')
+    user_item_info = request.json.get('user_item_info')
     cv_frame = data_uri_to_image(frame)
     gray_image = cv2.cvtColor(cv_frame, cv2.COLOR_BGR2GRAY)
     blur = cv2.bilateralFilter(gray_image,4,20,20)
@@ -32,7 +43,7 @@ def analyse():
     results = reader.readtext(equalized)
     results_array =[]
     for (bbox, text, prob) in results:
-        results_array.append(f"Detected text: {text} (Confidence: {prob})")
+        results_array.append(f"[{text} (Confidence: {prob})]")
 
 # Iterate through the results and format them
     #cv2.imshow("threash", thresh_image)
@@ -40,8 +51,18 @@ def analyse():
     #cv2.imshow("blur", blur)
     #cv2.imshow("orig", gray_image)
     #cv2.waitKey(0)
+    item=user_item_info
+    messages = [
+	{ "role": "system", "content": "You are a helpful nutritional information summarizer who is being used in a live application. You will be given the output of an OCR stage which has detected the text on a particular packaged food's label. You are tasked to give a comprehensive summary of the nutritional value, and ingredients if present. Comment on the health of the product.\nIf there are words you do not recognize, make educated guesses. At no point shall you refer to the inputted text or make users aware of any mistakes in character recognition. DO NOT REFER TO INPUT TEXT OR OCR " },
+	{ "role": "user", "content": f"{item}: {results_array}" }
+]
 
-    return str(results_array)
+    completion = client.chat.completions.create(
+    model="Qwen/Qwen2.5-Coder-32B-Instruct", 
+	messages=messages, 
+	max_tokens=500
+)
+    return str(completion.choices[0].message.content)
 
 
 def data_uri_to_image(data_uri):

static/index.js

@@ -104,7 +104,8 @@ function clear_pic(){
 
 function send_frame(dataUrl) {
   const dataToSend = {
-      key: dataUrl
+      key: dataUrl,
+      user_item_info: document.getElementById("user_item_info").value
   };
 
   fetch('/analyse', {

templates/index.html

@@ -29,6 +29,7 @@ box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.38); z-index:100">
       </div>
   </div>
   <center>
+  <input class="btn" type="text" id="user_item_info" placeholder="Enter item name...">
   <button class="btn" id="take-pic">Click</button>
   <button class="btn" id="clear-pic" style="display:none">Clear ✖</button>
   <button class="btn" id="send-pic" style="display:none">Send ➤</button>