Add application file

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,10 +1,15 @@
 FROM python:3.9
 
-COPY . .
+WORKDIR /app
 
-WORKDIR /
+RUN mkdir -p /app/.EasyOCR && chmod 777 /app/.EasyOCR
 
-RUN pip install --no-cache-dir --upgrade -r /requirements.txt
+ENV EASYOCR_MODULE_PATH="/app/.EasyOCR"
 
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+COPY requirements.txt .
 
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY ./*.py /app/
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,11 +1,9 @@
 ---
-title: Key Texts Image Finder
-emoji: 🏆
-colorFrom: purple
-colorTo: red
-sdk: gradio
-sdk_version: 5.22.0
-app_file: app.py
+title: Similarity
+emoji: 🌍
+colorFrom: indigo
+colorTo: gray
+sdk: docker
 pinned: false
 ---
 

app.py

@@ -1,17 +1,322 @@
-from fastapi import FastAPI
-from fastapi.middleware.cors import CORSMiddleware
+import gradio as gr
+import pandas as pd
+import torch
+from extract_text import extract_text_from_image
+from text_similarity import analyze_similarity
 
+def process_image(image, key_texts, similarity_threshold, fragment_threshold):
+    """Processes the image, extracts text, and analyzes similarities."""
+    try:
+        if image is None:
+            return "Please upload an image for analysis.", None, None, None, None, None
+        
+        if not key_texts.strip():
+            return "Please enter key texts for comparison.", None, None, None, None, None
+        
+        # Extract text from the image using the user's method
+        gpu_available = torch.cuda.is_available()
+        extracted_texts = extract_text_from_image(image, gpu_available)
+    
+        if isinstance(key_texts, str):
+            key_texts = [text.strip() for text in key_texts.split('\n') if text.strip()]
+        
+        # Process the analysis
+        results = analyze_similarity(
+            extracted_texts,
+            key_texts,
+            similarity_threshold=similarity_threshold/100,  # Convert percentage to decimal
+            fragment_threshold=fragment_threshold/100  # Convert percentage to decimal
+        )
+        
+        # Gerar relatório HTML
+        html_report = generate_html_report(results)
+        
+        # Gerar DataFrames
+        dfs = generate_results_dataframe(results)
+        
+        # Extrair DataFrames individuais (ou criar vazios se não existirem)
+        df_statistics = dfs.get("statistics", pd.DataFrame())
+        df_similar = dfs.get("similar", pd.DataFrame(columns=["Index", "Original Text", "Key Text", "Similarity"]))
+        df_fragments = dfs.get("fragments", pd.DataFrame(columns=["Index", "Original Text", "Key Text", "Similarity"]))
+        df_combined = dfs.get("combined", pd.DataFrame(columns=["Indices", "Text 1", "Text 2", "Combined Text", "Key Text", "Similarity"]))
+    
+        return html_report, df_statistics, df_similar, df_fragments, df_combined, extracted_texts, gpu_available
 
-app = FastAPI()
+    
+    except Exception as e:
+        return f"Erro ao processar: {str(e)}", None, None, None, None, None
+    
+def process_manual_input(texts, key_texts, similarity_threshold, fragment_threshold):
+    """Processes the user's manual text input."""
+    # Validate input
+    if not texts.strip() or not key_texts.strip():
+        return "Please enter texts for analysis and key texts for comparison.", None, None, None, None
+    
+    try:
+        # Process the analysis
+        results = analyze_similarity(
+            texts,
+            key_texts,
+            similarity_threshold=similarity_threshold/100,  # Convert percentage to decimal
+            fragment_threshold=fragment_threshold/100  # Convert percentage to decimal
+        )
+        
+        # Generate HTML report
+        html_report = generate_html_report(results)
+        
+        # Gerar DataFrames
+        dfs = generate_results_dataframe(results)
+        
+        # Extract individual DataFrames (or create empty ones if they don't exist)
+        df_statistics = dfs.get("statistics", pd.DataFrame())
+        df_similar = dfs.get("similar", pd.DataFrame(columns=["Index", "Original Text", "Key Text", "Similarity"]))
+        df_fragments = dfs.get("fragments", pd.DataFrame(columns=["Index", "Original Text", "Key Text", "Similarity"]))
+        df_combined = dfs.get("combined", pd.DataFrame(columns=["Indices", "Text 1", "Text 2", "Combined Text", "Key Text", "Similarity"]))
+        
+        return html_report, df_statistics, df_similar, df_fragments, df_combined
+    
+    except Exception as e:
+        return f"Erro ao processar: {str(e)}", None, None, None, None
 
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
+def generate_html_report(results):
+    """Generates an HTML report about the detected similarities."""
+    html = "<h2>Similarity Report</h2>"
+    
+    # General statistics
+    html += "<div padding: 15px; border-radius: 5px; margin-bottom: 20px;'>"
+    html += f"<p><b>Total texts analyzed:</b> {results['statistics']['total_analyzed']}</p>"
+    html += f"<p><b>Texts with detected similarity:</b> {results['statistics']['total_processed']}</p>"
+    html += "</div>"
+    
+     # Results table
+    html += "<h3>Detected Similarities</h3>"
+    
+    # Similar texts
+    if results["similar_texts"]:
+        html += "<h4>Direct Similar Texts</h4>"
+        html += "<table width='100%' style='border-collapse: collapse; margin-bottom: 20px;'>"
+        html += "<tr><th style='text-align: left; padding: 8px; border: 1px solid #ddd;'>Original Text</th><th style='text-align: left; padding: 8px; border: 1px solid #ddd;'>Key Text</th><th style='text-align: center; padding: 8px; border: 1px solid #ddd;'>Similarity</th></tr>"
+        
+        for item in results["similar_texts"]:
+            html += f"<tr><td style='padding: 8px; border: 1px solid #ddd;'>{item['text']}</td><td style='padding: 8px; border: 1px solid #ddd;'>{item['key_text']}</td><td style='text-align: center; padding: 8px; border: 1px solid #ddd;'>{item['similarity']:.2%}</td></tr>"
+        
+        html += "</table>"
+    
+    # Detected fragments
+    if results["fragments_detected"]:
+        html += "<h4>Text with Detected Fragments</h4>"
+        html += "<table width='100%' style='border-collapse: collapse; margin-bottom: 20px;'>"
+        html += "<tr><th style='text-align: left; padding: 8px; border: 1px solid #ddd;'>Original Text</th><th style='text-align: left; padding: 8px; border: 1px solid #ddd;'>Key Text</th><th style='text-align: center; padding: 8px; border: 1px solid #ddd;'>Similarity</th></tr>"
+        
+        for item in results["fragments_detected"]:
+            html += f"<tr><td style='padding: 8px; border: 1px solid #ddd;'>{item['text']}</td><td style='padding: 8px; border: 1px solid #ddd;'>{item['key_text']}</td><td style='text-align: center; padding: 8px; border: 1px solid #ddd;'>{item['similarity']:.2%}</td></tr>"
+        
+        html += "</table>"
+    
+    # Combined texts
+    if results["combined"]:
+        html += "<h4>Text that need to be combined</h4>"
+        html += "<table width='100%' style='border-collapse: collapse; margin-bottom: 20px;'>"
+        html += "<tr><th style='text-align: left; padding: 8px; border: 1px solid #ddd;'>Text 1</th><th style='text-align: left; padding: 8px; border: 1px solid #ddd;'>Text 2</th><th style='text-align: left; padding: 8px; border: 1px solid #ddd;'>Combination</th><th style='text-align: left; padding: 8px; border: 1px solid #ddd;'>Key Text</th><th style='text-align: center; padding: 8px; border: 1px solid #ddd;'>Similarity</th></tr>"
+        
+        for item in results["combined"]:
+            html += f"<tr><td style='padding: 8px; border: 1px solid #ddd;'>{item['texts'][0]}</td><td style='padding: 8px; border: 1px solid #ddd;'>{item['texts'][1]}</td><td style='padding: 8px; border: 1px solid #ddd;'>{item['combined_text']}</td><td style='padding: 8px; border: 1px solid #ddd;'>{item['key_text']}</td><td style='text-align: center; padding: 8px; border: 1px solid #ddd;'>{item['similarity']:.2%}</td></tr>"
+        
+        html += "</table>"
+    
+    if not (results["similar_texts"] or results["fragments_detected"] or results["combined"]):
+        html += "<p>No significant similarity found with the current parameters.</p>"
+    
+    return html
 
-@app.get("/", tags=["Home"])
-def api_home():
-    return {'detail': 'Welcome to FastAPI TextGen Tutorial!'}
+def generate_results_dataframe(results):
+    """Generates pandas DataFrames to visualize the results."""
+    dfs = {}
+    
+    # DataFrame for similar texts
+    if results["similar_texts"]:
+        data = [(item['index'], item['text'], item['key_text'], f"{item['similarity']:.2%}") 
+                for item in results["similar_texts"]]
+        dfs["similar"] = pd.DataFrame(data, columns=["Index", "Original Text", "Key Text", "Similarity"])
+    
+    # DataFrame for fragments
+    if results["fragments_detected"]:
+        data = [(item['index'], item['text'], item['key_text'], f"{item['similarity']:.2%}") 
+                for item in results["fragments_detected"]]
+        dfs["fragments"] = pd.DataFrame(data, columns=["Index", "Original Text", "Key Text", "Similarity"])
+    
+    # DataFrame for combined
+    if results["combined"]:
+        data = [(f"{item['indices'][0]},{item['indices'][1]}", 
+                  item['texts'][0], 
+                  item['texts'][1],
+                  item['combined_text'],
+                  item['key_text'], 
+                  f"{item['similarity']:.2%}") 
+                for item in results["combined"]]
+        dfs["combined"] = pd.DataFrame(data, columns=["Indices", "Text 1", "Text 2", 
+                                                        "Combined Text", "Key Text", "Similarity"])
+    
+    # Statistics DataFrame
+    data = [
+        ("Total analyzed", results["statistics"]["total_analyzed"]),
+        ("Total with similarity", results["statistics"]["total_processed"]),
+        ("Direct similarity", results["statistics"]["direct_similarity"]),
+        ("Fragments", results["statistics"]["fragments"]),
+        ("Combined", results["statistics"]["combined"])
+    ]
+    dfs["statistics"] = pd.DataFrame(data, columns=["Metric", "Value"])
+    
+    return dfs
+
+def generate_gradio():
+    with gr.Blocks(title="Text Similarity Detector") as demo:
+        gr.Markdown("# 🔍 Text Similarity Detector with Image Extraction")
+        gr.Markdown("""
+        This tool analyzes the similarity between texts extracted from an image and reference key texts.
+        It can identify:
+        - Direct similar texts
+        - Key text fragments within the texts
+        - Text combinations that match key texts
+        """)
+
+        with gr.Tabs() as tabs:
+            with gr.TabItem("Image Analysis"):
+                with gr.Row():
+                    with gr.Column(scale=1):  # Column for inputs on the left
+                        input_image = gr.Image(label="Upload an image to extract text", type="pil", height=600)
+                        key_texts_image = gr.Textbox(
+                                label="Key Texts for Comparison",
+                                placeholder="Paste your key texts here (one per line)",
+                                lines=5
+                            )
+                        # with gr.Row():
+                        #     key_texts_image = gr.Textbox(
+                        #         label="Key Texts for Comparison",
+                        #         placeholder="Paste your key texts here (one per line)",
+                        #         lines=5
+                        #     )
+
+                        #     min_similarity_per_key_image = gr.Textbox(
+                        #         label="Minimum Similarity for Each Key Text (%)",
+                        #         placeholder="Enter one value per line, matching the key texts",
+                        #         lines=5
+                        #     )
+
+                        with gr.Row():
+                            similarity_threshold_image = gr.Slider(
+                                label="Similarity Threshold (%)",
+                                minimum=50,
+                                maximum=100,
+                                value=70,
+                                step=1
+                            )
+                            fragment_threshold_image = gr.Slider(
+                                label="Fragment Similarity Threshold (%)",
+                                minimum=50,
+                                maximum=100,
+                                value=70,
+                                step=1
+                            )
+
+                        analyze_image_btn = gr.Button("Analyze Image", variant="primary")
+                    
+                    with gr.Column(scale=1):  # Column for outputs on the right
+                        gpu_available = gr.Checkbox(label="Used GPU")
+                        extracted_texts = gr.Textbox(label="Extracted Texts from the Image", lines=5)
+                        html_output = gr.HTML(label="Similarity Report")
+                        with gr.Tabs():
+                            with gr.TabItem("Statistics"):
+                                statistics_output = gr.Dataframe(label="Statistics")
+                            with gr.TabItem("Direct Similarity"):
+                                similar_texts_output = gr.Dataframe(label="Direct Similar Texts")
+                            with gr.TabItem("Fragments"):
+                                fragments_output = gr.Dataframe(label="Texts with Fragments")
+                            with gr.TabItem("Combined"):
+                                combined_output = gr.Dataframe(label="Combined Texts")
+
+            with gr.TabItem("Manual Analysis"):
+                with gr.Row():
+                    with gr.Column(scale=1):  # Column for inputs on the left
+                        input_texts = gr.Textbox(
+                            label="List of Texts for Analysis",
+                            placeholder="Paste your list of texts here (one per line)",
+                            lines=10
+                        )
+                        key_texts_input = gr.Textbox(
+                                label="Key Texts for Comparison",
+                                placeholder="Paste your key texts here (one per line)",
+                                lines=5
+                            )
+                        # with gr.Row():
+                        #     key_texts_input = gr.Textbox(
+                        #         label="Key Texts for Comparison",
+                        #         placeholder="Paste your key texts here (one per line)",
+                        #         lines=5
+                        #     )
+
+                        #     min_similarity_per_key_input = gr.Textbox(
+                        #         label="Minimum Similarity for Each Key Text (%)",
+                        #         placeholder="Enter one value per line, matching the key texts",
+                        #         lines=5
+                        #     )
+                                        
+                        with gr.Row():
+                            similarity_threshold = gr.Slider(
+                                label="Similarity Threshold (%)",
+                                minimum=50,
+                                maximum=100,
+                                value=70,
+                                step=1
+                            )
+                            fragment_threshold = gr.Slider(
+                                label="Fragment Similarity Threshold (%)",
+                                minimum=50,
+                                maximum=100,
+                                value=70,
+                                step=1
+                            )
+
+                        analyze_btn = gr.Button("Analyze Image", variant="primary")
+
+                    with gr.Column(scale=1):  # Column for outputs on the right
+                        html_output_manual = gr.HTML(label="Manual Similarity Report")
+                        with gr.Tabs():
+                            with gr.TabItem("Statistics"):
+                                statistics_output_manual = gr.Dataframe(label="Statistics")
+                            with gr.TabItem("Direct Similarity"):
+                                similar_texts_output_manual = gr.Dataframe(label="Direct Similar Texts")
+                            with gr.TabItem("Fragments"):
+                                fragments_output_manual = gr.Dataframe(label="Texts with Fragments")
+                            with gr.TabItem("Combined"):
+                                combined_output_manual = gr.Dataframe(label="Combined Texts")
+
+        # Connect the image processing function to the button
+        analyze_image_btn.click(
+            process_image,
+            inputs=[input_image, key_texts_image, similarity_threshold_image, fragment_threshold_image],
+            outputs=[html_output, statistics_output, similar_texts_output, fragments_output, combined_output, extracted_texts, gpu_available]
+        )
+
+        # Connect the manual text processing function to the button
+        analyze_btn.click(
+            process_manual_input,
+            inputs=[input_texts, key_texts_input, similarity_threshold, fragment_threshold],
+            outputs=[html_output_manual, statistics_output_manual, similar_texts_output_manual, fragments_output_manual, combined_output_manual]
+        )
+    
+    return demo
+
+#app = gr.mount_gradio_app(app, demo, path="/")
+
+if __name__ == "__main__":
+    generate_gradio.launch()
+
+# PORT = int(os.getenv("PORT", 7860))
+
+# if __name__ == "__main__":
+#     import uvicorn
+#     print(f"A arrancar na porta {PORT}...")
+#     uvicorn.run(app)
+
+    #demo.launch(server_name="0.0.0.0", server_port=7860)

extract_text.py

@@ -0,0 +1,29 @@
+import cv2
+import numpy as np
+import easyocr
+import torch
+
+# Inicializar EasyOCR
+device = "cuda" if torch.cuda.is_available() else "cpu"
+reader = easyocr.Reader(["en"], gpu=(device == "cuda"), verbose=False)
+
+def extract_text_from_image(img, gpu_available):
+    reader = easyocr.Reader(['en'], gpu=gpu_available, verbose=False)
+
+    img = np.array(img) 
+    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
+
+    # Resizing and blurring
+    scale_factor = 2
+    upscaled = cv2.resize(img, None, fx=scale_factor, fy=scale_factor, interpolation=cv2.INTER_LINEAR)
+    blur_img = cv2.blur(upscaled, (5, 5))
+    
+    all_text_found = []
+    text_ = reader.readtext(blur_img, detail=1, paragraph=False, text_threshold=0.3)
+
+    for t in text_:
+        bbox, text, score = t
+        if score > 0.1:  # Filter weak detections
+            all_text_found.append(text)
+
+    return all_text_found

main.py

@@ -0,0 +1,107 @@
+import logging
+import cv2
+import numpy as np
+import requests
+import torch
+import base64
+import gradio as gr
+from PIL import Image
+from io import BytesIO
+from fastapi import FastAPI
+from models import TextSimilarityRequest
+from extract_text import extract_text_from_image
+from text_similarity import analyze_similarity
+from app import generate_gradio
+
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+app = FastAPI()
+
+@app.get("/teste", tags=["Home"])
+def api_home():
+    return "oi"
+
+
+@app.post("/text_similarity", summary="Perform images text similarity", response_model=float, tags=["Text Similarities"])
+async def text_similarity(request: TextSimilarityRequest):
+    image_info = request.imageInfo
+    key_texts = request.keyTexts
+    similarity_threshold = request.similarityThreshold 
+    origin_id = image_info.originId
+
+    logging.info(f"Checking text similarity for main source with resource id {origin_id}")
+
+    image = load_image_url(image_info.source)
+
+    # Extract text from the image using the user's method
+    gpu_available = torch.cuda.is_available()
+    extracted_texts = extract_text_from_image(image, gpu_available)
+
+    results = analyze_similarity(
+            extracted_texts,
+            key_texts,
+            similarity_threshold=similarity_threshold/100,  # Convert percentage to decimal
+            fragment_threshold=100/100  # Convert percentage to decimal
+        )
+    
+    log_similarity_report(results, origin_id)
+    
+    total_texts = len(key_texts)
+    passed_texts = results["statistics"]["total_processed"]
+
+    percentage_passed = (passed_texts / total_texts) * 100
+
+    logging.info(f"Text similarity for main source with resource id {origin_id} is {percentage_passed}%")
+
+    return percentage_passed
+
+def log_similarity_report(results, originId):    
+    # General statistics
+    logging.info(f"[{originId}] Total texts analyzed: {results['statistics']['total_analyzed']}")
+    logging.info(f"[{originId}] Texts with detected similarity: {results['statistics']['total_processed']}")
+    
+    # Similar texts
+    if results["similar_texts"]:
+        logging.info(f"[{originId}] Direct Similar Texts Found: {len(results['similar_texts'])}")
+        for item in results["similar_texts"]:
+            logging.info(f"[{originId}] Similar Text: '{item['text']}' -> Key Text: '{item['key_text']}' with Similarity: {item['similarity']:.2%}")
+    
+    # Detected fragments
+    if results["fragments_detected"]:
+        logging.info(f"[{originId}] Fragments Detected: {len(results['fragments_detected'])}")
+        for item in results["fragments_detected"]:
+            logging.info(f"[{originId}] Fragment: '{item['text']}' -> Key Text: '{item['key_text']}' with Similarity: {item['similarity']:.2%}")
+    
+    # Combined texts
+    if results["combined"]:
+        logging.info(f"[{originId}] Texts to be Combined: {len(results['combined'])}")
+        for item in results["combined"]:
+            logging.info(f"[{originId}] Combined Text: '{item['combined_text']}' -> Key Text: '{item['key_text']}' with Similarity: {item['similarity']:.2%}")
+    
+    # If no significant similarity found
+    if not (results["similar_texts"] or results["fragments_detected"] or results["combined"]):
+        logging.info(f"[{originId}] No significant similarity found.")
+    
+    # Statistics
+    logging.info(f"[{originId}] Direct similarity: {results['statistics']['direct_similarity']}")
+    logging.info(f"[{originId}] Fragments: {results['statistics']['fragments']}")
+    logging.info(f"[{originId}] Combined: {results['statistics']['combined']}")
+
+def load_image_url(source):
+    Image.MAX_IMAGE_PIXELS = None
+
+    if source.startswith('http'):
+        response = requests.get(source)
+        img = np.asarray(bytearray(response.content), dtype=np.uint8)
+        img = cv2.imdecode(img, cv2.IMREAD_GRAYSCALE)
+    else:
+        img = base64.b64decode(source)
+        img = Image.open(BytesIO(img))
+        img = np.array(img)
+        img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+
+    return img
+
+@app.on_event("startup")
+async def startup_event():
+    gr.mount_gradio_app(app, generate_gradio(), path="/")

models.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+from typing import List
+
+class RequestModel(BaseModel):
+    originId: int
+    source: str
+
+class TextSimilarityRequest(BaseModel):
+    imageInfo: RequestModel
+    keyTexts: List[str]
+    similarityThreshold: float 

requirements.txt

@@ -1,7 +1,11 @@
-fastapi==0.99.1
-uvicorn
+numpy
 requests
-pydantic==1.10.12
-langchain
-clarifai
-Pillow
+fastapi
+pydantic
+scikit-image
+pillow
+uvicorn
+opencv-python-headless
+torch
+easyocr
+gradio

text_similarity.py

@@ -0,0 +1,125 @@
+import re
+from difflib import SequenceMatcher
+from collections import defaultdict
+
+def extract_special_characters(text):
+    """Extracts all unique special characters from a list of texts."""
+    characters = re.findall(r'[^\w\s]', text)  # Finds non-alphanumeric and non-space characters
+    return ''.join(characters)
+
+def clean_text(text, keep=""):
+    """Removes special characters except those specified in 'keep', and converts to lowercase."""
+    pattern = rf'[^\w\s{re.escape(keep)}]'
+    return re.sub(pattern, '', text.lower())
+
+def text_similarity(text, key_text):
+    """Calculates the similarity between two texts using SequenceMatcher."""
+    return SequenceMatcher(None, text, key_text).ratio()
+
+def detect_fragments(text, key_texts, threshold=0.7):
+    """Checks if a text contains fragments of key texts."""
+    for key_text in key_texts:
+        characters_to_not_clean = extract_special_characters(key_text)
+        words = clean_text(text, characters_to_not_clean).split()
+
+        key_words = key_text.split()
+        
+        # If the text is too short, we can't make an effective sliding window
+        if len(words) < len(key_words):
+            similarity = text_similarity(text, key_text)
+            if similarity >= threshold:
+                return True, key_text, similarity
+            continue
+        
+        # Sliding window to compare word sequences
+        for i in range(len(words) - len(key_words) + 1):
+            fragment = " ".join(words[i:i+len(key_words)])
+            similarity = text_similarity(fragment, key_text)
+            if similarity >= threshold:
+                return True, key_text, similarity
+    return False, None, 0
+
+def analyze_similarity(text_list, key_texts, similarity_threshold=0.7, fragment_threshold=0.7):
+    """
+    Analyzes the similarity between a list of texts and key texts.
+    Returns a detailed report on the similarities found.
+    """
+    results = {
+        "similar_texts": [],
+        "fragments_detected": [],
+        "combined": [],
+        "statistics": defaultdict(int)
+    }
+    
+    processed_texts = set()
+    
+    # Check direct similarity
+    for i, text in enumerate(text_list):
+        if not text.strip():
+            continue
+            
+        for key_text in key_texts:
+            if not key_text.strip():
+                continue
+                
+            similarity = text_similarity(text, key_text)
+            if similarity >= similarity_threshold:
+                results["similar_texts"].append({
+                    "index": i,
+                    "text": text,
+                    "key_text": key_text,
+                    "similarity": similarity
+                })
+                results["statistics"]["direct_similarity"] += 1
+                processed_texts.add(i)
+    
+    # Check fragments
+    # for i, text in enumerate(text_list):
+    #     if i in processed_texts or not text.strip():
+    #         continue
+            
+    #     has_fragment, key_text, similarity = detect_fragments(text, key_texts, fragment_threshold)
+    #     if has_fragment:
+    #         results["fragments_detected"].append({
+    #             "index": i,
+    #             "text": text,
+    #             "key_text": key_text,
+    #             "similarity": similarity
+    #         })
+    #         results["statistics"]["fragments"] += 1
+    #         processed_texts.add(i)
+    
+    # Check texts that can be combined
+    for i in range(len(text_list)):
+        if i in processed_texts or not text_list[i].strip():
+            continue
+            
+        for j in range(i+1, len(text_list)):
+            if j in processed_texts or not text_list[j].strip():
+                continue
+                
+            combined_text = text_list[i] + " " + text_list[j]
+            for key_text in key_texts:
+                if not key_text.strip():
+                    continue
+                    
+                similarity = text_similarity(combined_text, key_text)
+                if similarity >= similarity_threshold:
+                    results["combined"].append({
+                        "indices": [i, j],
+                        "texts": [text_list[i], text_list[j]],
+                        "combined_text": combined_text,
+                        "key_text": key_text,
+                        "similarity": similarity
+                    })
+                    results["statistics"]["combined"] += 1
+                    processed_texts.add(i)
+                    processed_texts.add(j)
+                    break
+    
+    # Calculate overall statistics
+    valid_texts = sum(1 for text in text_list if text.strip())
+    results["statistics"]["total_analyzed"] = valid_texts
+    results["statistics"]["total_processed"] = len(processed_texts)
+    
+    return results