Upload 26 files

app.py

@@ -1,12 +1,22 @@
 import streamlit as st
 import tensorflow as tf
 from button_click_alt import find_order_id
+from button_click_alt import find_order_id_similarity
+from flann import generate_images, flann_matching, flann_matching_3, flann_matching_alt, flann_matching_4
+import cv2
+import numpy as np
+import pandas as pd
+import plotly.express as px
+import os
+
+FONTS_FOLDER = "fonts"
+NUM_IMAGES_PER_FONT = 5
 
 def main():
     st.set_page_config(page_title='Order ID Finder', layout='wide')
     st.title('OCR + Font type demo')
 
-    tabs = st.tabs(["Intro", "find order"])
+    tabs = st.tabs(["Intro", "Find Order", "Try FLANN Matching"])
 
     with tabs[0]:
         col1, col2 = st.columns([1, 2])
@@ -47,6 +57,8 @@ def main():
         colab_link = '[<img src="https://colab.research.google.com/assets/colab-badge.svg">](https://colab.research.google.com/drive/1tq35g7ym1c73uDlAcy2KChIXsqlNY-RL?usp=sharing)'
         st.markdown(colab_link, unsafe_allow_html=True)
 
+
+
     with tabs[1]:
         st.write('## Find Order')
         st.write('This is the find order tab')
@@ -55,19 +67,88 @@ def main():
             st.write('## Input')
             uploaded_file = st.file_uploader('Upload the image file (PNG or JPG)', type=['png', 'jpg'], help='help')
             input_file = st.file_uploader('Upload the input file (TXT)', type=['txt'], help='text file containing order id, text, font type. in that order')
-            with st.expander('Settings'):
+            with st.expander('OCR Settings'):
                 ocre = st.selectbox('OCR Engine', ['Hive', 'Tesseract'])
                 img_processing = st.selectbox('Image preprocessing', ['Gray Scaling', 'Thresholding, Denoising, Binarization, Skew Correction', 'Adaptive Thresholding, Morphological Operations, CCA'])
-                cnn_model = st.selectbox('Neural Network Model', ['CNN-MaxPool-Dense-Dropout', 'BatchNorm-CNN-MaxPool-Dense-Dropout'])
-    
-        if st.button('Find Order ID') and uploaded_file and input_file:
-            st.write('## Output')
-            model = tf.keras.models.load_model('model.h5')
-            result = find_order_id(uploaded_file, input_file, model, ocre)
-            if result['status'] == 'success':
-                st.success(result['message'])
-            elif result['status'] == 'warning':
-                st.warning(result['message'])
+                
+            with st.expander('Other Settings'):
+                cnn_model = st.selectbox('Font Classification Model', ['CNN-MaxPool-Dense-Dropout', 'BatchNorm-CNN-MaxPool-Dense-Dropout'])
+                similarity_method = st.selectbox('Similarity Check', ['jaccard_similarity', 'exact_match'])
+        
+        col1, col2 = st.columns([1, 2])
+        with col1:
+            if st.button('Find Order ID by OCR + font type') and uploaded_file and input_file:
+                st.write('## Output')
+                model = tf.keras.models.load_model('model.h5')
+                result = find_order_id(uploaded_file, input_file, model, ocre)
+                if result['status'] == 'success':
+                    st.success(result['message'])
+                elif result['status'] == 'warning':
+                    st.warning(result['message'])
+        with col2:
+            if st.button('Find Order ID by OCR + similarity check') and uploaded_file and input_file:
+                st.write('## Output')
+                result = find_order_id_similarity(uploaded_file, input_file, similarity_method, ocre)
+                if result['status'] == 'success':
+                    st.success(result['message'])
+                elif result['status'] == 'warning':
+                    st.warning(result['message'])
+
+    with tabs[2]:
+        st.write('## Try FLANN Matching')
+        text_input = st.text_input("Enter your text:")
+        upload_image = st.file_uploader("Choose an image:", type=["jpg", "jpeg", "png"])
+        col1, col2, col3 = st.columns(3)
+        with col1:
+            num_trees = st.slider("Number of trees:", 1, 20, 5)
+        with col2:
+            num_checks = st.slider("Number of checks:", 1, 200, 50)
+        with col3:
+            matching_methods = ["FLANN with SIFT descriptor and ratio test", "FLANN with SIFT descriptor and KNN matching", "FLANN with SIFT descriptor, RANSAC homography estimation, and ORB descriptor", "Basic FLANN"]
+            selected_method = st.selectbox("Select FLANN matching method:", matching_methods)
+        if st.button("Generate Images"):
+            if text_input:
+                generated_images = generate_images(text_input)
+                st.write(f"{len(generated_images)} images generated ({NUM_IMAGES_PER_FONT} per font) for {len(os.listdir(FONTS_FOLDER))} font types.")
+                with st.expander("Generated Images"):
+                    for img, font_file in generated_images:
+                        st.image(img, caption=font_file)
+            else:
+                st.warning("Please enter some text before generating images.")
+        if upload_image:
+            query_image = cv2.imdecode(np.fromstring(upload_image.read(), np.uint8), cv2.IMREAD_UNCHANGED)
+            st.image(query_image, caption="Uploaded Image")
+        if st.button("Match"):
+            generated_images = generate_images(text_input)
+            if selected_method == "FLANN with SIFT descriptor and ratio test":
+                matching_results = flann_matching_alt(generated_images, query_image, num_trees, num_checks)
+            elif selected_method == "FLANN with SIFT descriptor and KNN matching":
+                matching_results = flann_matching(generated_images, query_image, num_trees, num_checks)
+            elif selected_method == "FLANN with SIFT descriptor, RANSAC homography estimation, and ORB descriptor":
+                matching_results = flann_matching_3(generated_images, query_image, num_trees, num_checks)
+            else:
+                matching_results = flann_matching_4(generated_images, query_image, num_trees, num_checks)
+            matching_percentages = []
+            with col1:
+                with st.expander("Matching Images"):
+                    for r, f, p in matching_results:
+                        st.image(r, caption=f"Matching result for {f}, Matches: {p:.2f}%")
+            for r, font_file, p in matching_results:
+                matching_percentages.append((font_file, p))
+            df = pd.DataFrame(matching_percentages, columns=['Font Type', 'Match Percent'])
+            avg_df = df.groupby('Font Type').mean()
+            with col2:
+                with st.expander("All Results"):
+                    st.write("Overall matching percentages for each font type:")
+                    st.table(df)
+                st.write("Average matching percentage for each font type:")
+                st.table(avg_df)
+                fig = px.bar(avg_df.reset_index(), x='Font Type', y='Match Percent')
+                fig.update_layout(title='Average Matching Percentages by Font Type')
+                st.plotly_chart(fig)
+            max_match_font = avg_df['Match Percent'].idxmax()
+            st.success(f"The most likely font type is: {max_match_font}")
+
 
 if __name__ == '__main__':
     main()

button_click_alt.py

@@ -80,3 +80,57 @@ def find_order_id(uploaded_file, input_file, model, ocre):
                 }
 
     return result
+
+def jaccard_similarity(s1, s2):
+    set1 = set(s1.split())
+    set2 = set(s2.split())
+    intersection = len(set1.intersection(set2))
+    union = len(set1.union(set2))
+    return intersection / union
+
+def find_order_id_similarity(uploaded_file, input_file, similarity_method, ocre):
+    if ocre == 'Hive':
+        uploaded_image = Image.open(uploaded_file)
+        text = infer_text(uploaded_image)
+    else:
+        rotated = preprocess_image(uploaded_file)
+        text = pytesseract.image_to_string(rotated)
+
+    with input_file as file:
+        file_contents = file.read().decode()
+        lines = file_contents.split('\n')
+
+        if similarity_method == 'exact_match':
+            for line in lines:
+                order_id, name, font = line.strip().split(',')
+                if name.strip() == text.strip():
+                    result = {
+                        'status': 'success',
+                        'message': f'Detected Text: {text.strip()}\n, Order ID: {order_id}'
+                    }
+                    return result
+            message = f'Detected Text: {text.strip()}\n, Could not find the Order ID.'
+            result = {'status': 'error', 'message': message}
+            return result
+
+        elif similarity_method == 'jaccard_similarity':
+            possible_order_ids = []
+            for line in lines:
+                order_id, name, font = line.strip().split(',')
+                jaccard_score = jaccard_similarity(name.strip(), text.strip())
+                if jaccard_score >= 0.8:
+                    result = {
+                        'status': 'success',
+                        'message': f'Detected Text: {text.strip()}\n, Order ID: {order_id}'
+                    }
+                    return result
+                elif jaccard_score >= 0.5:
+                    possible_order_ids.append(order_id)
+            if len(possible_order_ids) > 0:
+                message = f'Detected Text: {text.strip()}\n, Possible Order IDs: {",".join(possible_order_ids)}'
+                result = {'status': 'warning', 'message': message}
+                return result
+            else:
+                message = f'Detected Text: {text.strip()}\n, Could not find the Order ID.'
+                result = {'status': 'error', 'message': message}
+                return result

flann.py

@@ -0,0 +1,201 @@
+from PIL import Image, ImageDraw, ImageFont
+import numpy as np
+import cv2
+import os
+
+FONTS_FOLDER = "fonts"
+NUM_IMAGES_PER_FONT = 5
+
+def generate_images(text):
+    images = []
+    for font_file in os.listdir(FONTS_FOLDER):
+        font_path = os.path.join(FONTS_FOLDER, font_file)
+        for i in range(NUM_IMAGES_PER_FONT):
+            img = generate_text_image(text, font_path)
+            images.append((img, font_file))
+    return images
+
+def generate_text_image(text, font_path, fontsize=None):
+    if not fontsize:
+        fontsize = int(np.random.normal(loc=50, scale=10))
+    font = ImageFont.truetype(font_path, fontsize)
+    text_size = font.getsize(text)
+    img = Image.new('RGB', text_size, color='black')
+    draw = ImageDraw.Draw(img)
+    draw.text((0, 0), text, font=font, fill='white')
+    noise = np.random.normal(loc=0, scale=10, size=(img.size[1], img.size[0]))[..., np.newaxis]
+    noise = np.tile(noise, [1, 1, 3])
+    img = Image.fromarray(np.clip(np.array(img) + noise, 0, 255).astype(np.uint8), 'RGB')
+    return np.array(img)
+
+def flann_matching_alt(generated_images, query_image, num_trees=5, num_checks=50):
+    query_image_gray = cv2.cvtColor(query_image, cv2.COLOR_BGR2GRAY)
+    generated_images_gray = []
+    for img, _ in generated_images:
+        generated_images_gray.append(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY))
+
+    sift = cv2.SIFT_create()
+    index_params = dict(algorithm=0, trees=num_trees)
+    search_params = dict(checks=num_checks)
+    flann = cv2.FlannBasedMatcher(index_params, search_params)
+
+    query_kp, query_desc = sift.detectAndCompute(query_image_gray, None)
+
+    if query_desc is None:
+        return []
+
+    matching_results = []
+    for i, (img, font_file) in enumerate(generated_images):
+        kp, desc = sift.detectAndCompute(generated_images_gray[i], None)
+        if desc is not None:
+            matches = flann.knnMatch(query_desc, desc, k=2)
+            good_matches = []
+            for m, n in matches:
+                if m.distance < 0.75 * n.distance:
+                    good_matches.append(m)
+            matching_img = cv2.drawMatches(query_image_gray, query_kp, generated_images_gray[i], kp, good_matches, None, flags=2)
+            # Calculate percentage match
+            num_query_kp = len(query_kp)
+            num_matches = len(good_matches)
+            match_percent = 100 * num_matches / num_query_kp
+            matching_results.append((matching_img, font_file, match_percent))
+
+    return matching_results
+
+def flann_matching(generated_images, query_image, num_trees=5, num_checks=50):
+    query_image_gray = cv2.cvtColor(query_image, cv2.COLOR_BGR2GRAY)
+    generated_images_gray = []
+    for img, _ in generated_images:
+        generated_images_gray.append(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY))
+    
+    sift = cv2.SIFT_create()
+    index_params = dict(algorithm=0, trees=num_trees)
+    search_params = dict(checks=num_checks)
+    flann = cv2.FlannBasedMatcher(index_params, search_params)
+    
+    query_kp, query_desc = sift.detectAndCompute(query_image_gray, None)
+    generated_kp = []
+    generated_desc = []
+    for img in generated_images_gray:
+        kp, desc = sift.detectAndCompute(img, None)
+        generated_kp.append(kp)
+        generated_desc.append(desc)
+    
+    matching_results = []
+    for i, (img, font_file) in enumerate(generated_images):
+        matches = flann.knnMatch(query_desc, generated_desc[i], k=2)
+        good_matches = []
+        for m, n in matches:
+            if m.distance < 0.75*n.distance:
+                good_matches.append([m])
+        matching_img = cv2.drawMatchesKnn(query_image_gray, query_kp, img, generated_kp[i], good_matches, None, flags=2)
+        # Calculate percentage match
+        num_query_kp = len(query_kp)
+        num_matches = len(good_matches)
+        match_percent = 100*num_matches/num_query_kp
+        matching_results.append((matching_img, font_file, match_percent))
+        
+    return matching_results
+
+def flann_matching_3(generated_images, query_image, num_trees=5, num_checks=50):
+    query_image_gray = cv2.cvtColor(query_image, cv2.COLOR_BGR2GRAY)
+    generated_images_gray = [cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) for img, _ in generated_images]
+    
+    sift = cv2.SIFT_create()
+    index_params = dict(algorithm=0, trees=num_trees)
+    search_params = dict(checks=num_checks)
+    flann = cv2.FlannBasedMatcher(index_params, search_params)
+    
+    query_kp, query_desc = sift.detectAndCompute(query_image_gray, None)
+    
+    if query_desc is None:
+        return []
+    
+    matching_results = []
+    for i, (img, font_file) in enumerate(generated_images):
+        kp, desc = sift.detectAndCompute(generated_images_gray[i], None)
+        
+        if desc is None:
+            continue
+        
+        matches = flann.knnMatch(query_desc, desc, k=2)
+        good_matches = []
+        for m, n in matches:
+            if m.distance < 0.75 * n.distance:
+                good_matches.append(m)
+        
+        if len(good_matches) < 10:
+            continue
+        
+        src_pts = np.float32([query_kp[m.queryIdx].pt for m in good_matches]).reshape(-1, 1, 2)
+        dst_pts = np.float32([kp[m.trainIdx].pt for m in good_matches]).reshape(-1, 1, 2)
+        M, _ = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
+        
+        if M is None:
+            continue
+        
+        h, w = query_image_gray.shape
+        dst_img = cv2.warpPerspective(img, M, (w, h))
+        dst_gray = cv2.cvtColor(dst_img, cv2.COLOR_BGR2GRAY)
+        
+        orb = cv2.ORB_create()
+        kp1, desc1 = sift.detectAndCompute(query_image_gray, None)
+        kp2, desc2 = sift.detectAndCompute(dst_gray, None)
+        
+        if desc1 is None or desc2 is None:
+            continue
+        
+        matches = flann.knnMatch(desc1, desc2, k=2)
+        good_matches = []
+        for m, n in matches:
+            if m.distance < 0.75 * n.distance:
+                good_matches.append(m)
+        
+        if len(good_matches) < 10:
+            continue
+        
+        matching_img = cv2.drawMatches(query_image_gray, kp1, dst_gray, kp2, good_matches, None, flags=2)
+        # Calculate percentage match
+        num_query_kp = len(kp1)
+        num_matches = len(good_matches)
+        match_percent = 100 * num_matches / num_query_kp
+        matching_results.append((matching_img, font_file, match_percent))
+        
+    return matching_results
+
+def flann_matching_4(generated_images, query_image, num_trees=5, num_checks=50):
+    query_image_gray = cv2.cvtColor(query_image, cv2.COLOR_BGR2GRAY)
+    generated_images_gray = [cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) for img, _ in generated_images]
+
+    sift = cv2.SIFT_create()
+    index_params = dict(algorithm=0, trees=num_trees)
+    search_params = dict(checks=num_checks)
+    flann = cv2.FlannBasedMatcher(index_params, search_params)
+
+    query_kp, query_desc = sift.detectAndCompute(query_image_gray, None)
+
+    if query_desc is None:
+        return []
+
+    matching_results = []
+    for img, font_file in generated_images:
+        generated_image_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+        kp, desc = sift.detectAndCompute(generated_image_gray, None)
+
+        if desc is None:
+            continue
+
+        matches = flann.knnMatch(query_desc, desc, k=2)
+        good_matches = []
+        for m, n in matches:
+            if m.distance < 0.75 * n.distance:
+                good_matches.append(m)
+        matching_img = cv2.drawMatches(query_image_gray, query_kp, generated_image_gray, kp, good_matches, None, flags=2)
+        
+        # Calculate percentage match
+        num_query_kp = len(query_kp)
+        num_matches = len(good_matches)
+        match_percent = 100 * num_matches / num_query_kp
+        matching_results.append((matching_img, font_file, match_percent))
+
+    return matching_results

process.py

@@ -35,7 +35,7 @@ def preprocess_image(image_file):
     return rotated
 
 
-#not yet working
+
 def preprocess_image_high(image_file):
     img = cv2.imread(image_file)
     gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)