Added Semantic Textual Similarity

app.py

@@ -1,61 +1,106 @@
-from pathlib import Path
+import multiprocessing
+import threading
 import gradio as gr
-import pandas as pd
-import polars as pl
-from datasets import Dataset
-from sentence_transformers import SentenceTransformer
-from sentence_transformers.util import paraphrase_mining
-import torch
+from mining import mining
+from sts import sts
+from utils import getDataFrame, save_to_csv, delete_folder_periodically
 
+CONCURRENCY_LIMIT = 5
 
-def upload_file(filepath):
-    name = Path(filepath).name
-    return [gr.UploadButton(visible=False), gr.DownloadButton(label=f"Download {name}", value=filepath, visible=True)]
+with gr.Blocks() as demo:
+    with gr.Tab("Paraphrase Mining"):
+        with gr.Row():
+            gr.Markdown(
+                "### Paraphrase mining is the task of finding paraphrases (texts with identical / similar meaning) in a large corpus of sentences")
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("#### sentences")
 
+                upload_button_sentences = gr.UploadButton(label="upload sentences csv", file_types=['.csv'],
+                                                          file_count="single")
+                output_data_sentences = gr.Dataframe(headers=["text"], col_count=1, label="sentences data")
 
-def getData(path):
-    #data = Dataset.from_csv(path, column_names=["text"])
-    data = Dataset.from_pandas(pd.read_csv(path, on_bad_lines='skip', names=["text"]))
-    device = "cuda" if torch.cuda.is_available() else "cpu"
-    model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2",
-                                backend="openvino",
-                                device=device,
-                                trust_remote_code=True)
+                upload_button_sentences.upload(fn=getDataFrame, inputs=upload_button_sentences,
+                                               outputs=output_data_sentences, concurrency_limit=CONCURRENCY_LIMIT)
 
-    paraphrases = paraphrase_mining(
-        model,
-        data["text"],
-        corpus_chunk_size=len(data),
-        show_progress_bar=True,
-        batch_size=1024,
-        max_pairs=len(data) ** 2
-    )
+        with gr.Row():
+            with gr.Column():
+                submit_button_mining = gr.Button("Submit", variant="primary")
+        with gr.Row():
+            with gr.Column():
+                output_mining = gr.Dataframe(headers=["score", "sentence_1", "sentence_2"], type="polars",
+                                             label="Mining")
 
-    df_pd = pd.DataFrame(paraphrases)
-    df = pl.from_pandas(df_pd)
-    df = df.rename({"0": "score", "1": "sentence_1", "2": "sentence_2"})
+                submit_button_mining.click(
+                    fn=mining,
+                    inputs=upload_button_sentences,
+                    outputs=output_mining
+                )
 
-    union_df = pl.DataFrame(data.to_pandas())
+                download_button = gr.Button("Download Results as CSV", variant="huggingface")
+                download_file = gr.File(label="Downloadable File")
 
-    df = df.with_columns([
-        pl.col("score").round(3).cast(pl.Float32),
-        union_df.select(pl.col("text")).to_series()[df["sentence_1"].cast(pl.Int32)].alias("sentence_1"),
-        union_df.select(pl.col("text")).to_series()[df["sentence_2"].cast(pl.Int32)].alias("sentence_2"),
-    ]).filter(pl.col("score") > 0.96).sort(["score"], descending=True)
+                download_button.click(
+                    fn=save_to_csv,
+                    inputs=output_mining,
+                    outputs=download_file
+                )
 
-    data = pl.from_arrow(data.data.table)
+    with gr.Tab("Semantic Textual Similarity"):
+        with gr.Row():  # Row for the title
+            gr.Markdown(
+                "### Semantic Textual Similarity (STS), we want to produce embeddings for all texts involved and calculate the similarities between them")
+        with gr.Row():  # First row of two columns
+            with gr.Column():
+                gr.Markdown("#### sentences 1")
+                upload_button_sentences1 = gr.UploadButton(label="upload sentences 1 csv", file_types=['.csv'],
+                                                           file_count="single")
+                output_data_sentences1 = gr.Dataframe(headers=["text"], col_count=1, label="sentences 1 data")
 
-    return [data, df]
+                upload_button_sentences1.upload(fn=getDataFrame, inputs=upload_button_sentences1,
+                                                outputs=output_data_sentences1, concurrency_limit=CONCURRENCY_LIMIT)
 
+            with gr.Column():
+                gr.Markdown("#### sentences 2")
+                upload_button_sentences2 = gr.UploadButton(label="upload sentences 2 csv", file_types=['.csv'],
+                                                           file_count="single")
+                output_data_sentences2 = gr.Dataframe(headers=["text"], col_count=1, label="sentences 2 data")
 
-with gr.Blocks() as demo:
-    with gr.Column():
-        upload_button = gr.UploadButton(label="upload csv", file_types=['.csv'], file_count="single")
-        output_data = gr.Dataframe(headers=["text"], col_count=1, label="Uploaded Data")
-        output_paraphrases = gr.Dataframe(headers=["score", "sentence_1", "sentence_2"], type="polars",
-                                          label="Paraphrase Mining Results")
+                upload_button_sentences2.upload(fn=getDataFrame, inputs=upload_button_sentences2,
+                                                outputs=output_data_sentences2, concurrency_limit=CONCURRENCY_LIMIT)
+
+        with gr.Row():
+            with gr.Column():
+                submit_button_sts = gr.Button("Submit", variant="primary")
+
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("#### STS Results")
+
+                output_sts = gr.Dataframe(headers=["score", "sentence_1", "sentence_2"], type="polars",
+                                          label="Semantic Textual Similarit")
 
-        upload_button.upload(fn=getData, inputs=upload_button, outputs=[output_data, output_paraphrases])
+                submit_button_sts.click(
+                    fn=sts,
+                    inputs=[upload_button_sentences1, upload_button_sentences2],
+                    outputs=output_sts
+                )
+
+                download_button = gr.Button("Download Results as CSV", variant="huggingface")
+                download_file = gr.File(label="Downloadable File")
+
+                download_button.click(
+                    fn=save_to_csv,
+                    inputs=output_sts,
+                    outputs=download_file
+                )
 
 if __name__ == "__main__":
+    multiprocessing.set_start_method("spawn")
+
+    folder_path = "data"
+    thread = threading.Thread(target=delete_folder_periodically, args=(folder_path, 1800), daemon=True)
+    thread.start()
+
+    print(gr.__version__)
     demo.launch()

mining.py

@@ -0,0 +1,46 @@
+import time
+import pandas as pd
+import polars as pl
+import torch
+from datasets import Dataset
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.util import paraphrase_mining
+
+def mining(path):
+    st = time.time()
+    data = Dataset.from_pandas(pd.read_csv(path, on_bad_lines='skip', header=0, names=["text"]))
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model = SentenceTransformer(
+        "sentence-transformers/all-MiniLM-L6-v2",
+        backend="openvino",
+        model_kwargs={"file_name": "openvino/openvino_model.xml"},
+        device=device,
+        trust_remote_code=True,
+    )
+
+    paraphrases = paraphrase_mining(
+        model,
+        data["text"],
+        corpus_chunk_size=len(data),
+        show_progress_bar=True,
+        batch_size=1024,
+        max_pairs=len(data) ** 2,
+    )
+
+    df_pd = pd.DataFrame(paraphrases)
+    df = pl.from_pandas(df_pd)
+    df = df.rename({"0": "score", "1": "sentence_1", "2": "sentence_2"})
+
+    union_df = pl.DataFrame(data.to_pandas())
+
+    df = df.with_columns([
+        pl.col("score").round(3).cast(pl.Float32),
+        union_df.select(pl.col("text")).to_series()[df["sentence_1"].cast(pl.Int32)].alias("sentence_1"),
+        union_df.select(pl.col("text")).to_series()[df["sentence_2"].cast(pl.Int32)].alias("sentence_2"),
+    ]).filter(pl.col("score") > 0.96).sort(["score"], descending=True)
+
+    elapsed_time = time.time() - st
+    print('Execution time:', time.strftime("%H:%M:%S", time.gmtime(elapsed_time)))
+
+    return df

requirements.txt

@@ -4,3 +4,4 @@ pandas
 polars
 datasets
 sentence-transformers[openvino,onnx-gpu,onnx]
+gradio

sts.py

@@ -0,0 +1,56 @@
+import time
+import pandas as pd
+import polars as pl
+import torch
+from datasets import Dataset
+from sentence_transformers import SentenceTransformer
+
+def sts(data1, data2):
+    st = time.time()
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model = SentenceTransformer(
+        "sentence-transformers/all-MiniLM-L6-v2",
+        backend="openvino",
+        model_kwargs={"file_name": "openvino/openvino_model.xml"},
+        device=device,
+        trust_remote_code=True,
+    )
+
+    sentences1 = Dataset.from_pandas(pd.read_csv(data1, on_bad_lines='skip', header=0, names=["text"]))
+    sentences2 = Dataset.from_pandas(pd.read_csv(data2, on_bad_lines='skip', header=0, names=["text"]))
+
+    embeddings1 = model.encode(sentences1["text"], normalize_embeddings=True, batch_size=1024,
+                               show_progress_bar=True)
+    embeddings2 = model.encode(sentences2["text"], normalize_embeddings=True, batch_size=1024,
+                               show_progress_bar=True)
+
+    similarity_matrix = model.similarity(embeddings1, embeddings2)
+
+    df_pd = pd.DataFrame(similarity_matrix)
+    dfi = df_pd.__dataframe__()
+    df = pl.from_dataframe(dfi)
+
+    df_matrix_with_index = df.with_row_index(name="row_index").with_columns(pl.col("row_index").cast(pl.UInt64))
+    df_long = df_matrix_with_index.unpivot(index="row_index", variable_name="column_index",
+                                           value_name="score").with_columns(pl.col("column_index").cast(pl.UInt64))
+    df_sentences1 = pl.DataFrame(sentences1.to_pandas()).with_row_index(name="row_index").with_columns(
+        pl.col("row_index").cast(pl.UInt64))
+    df_sentences2 = pl.DataFrame(sentences2.to_pandas()).with_row_index(name="column_index").with_columns(
+        pl.col("column_index").cast(pl.UInt64))
+
+    df_long = (df_long
+               .with_columns([pl.col("score").round(4).cast(pl.Float32)])  # Ensure column_index is UInt32
+               .join(df_sentences1, on="row_index")
+               .join(df_sentences2, on="column_index"))
+
+    df_long = df_long.rename({
+        "text": "setences1",
+        "text_right": "sentences2",
+    })
+
+    elapsed_time = time.time() - st
+    print('Execution time:', time.strftime("%H:%M:%S", time.gmtime(elapsed_time)))
+
+    return df_long.select(["score", "setences1", "sentences2"]).filter(pl.col("score") > 0.96).sort(["score"],
+                                                                                                    descending=True)

utils.py

@@ -0,0 +1,25 @@
+import os
+import uuid
+import shutil
+import pandas as pd
+import polars as pl
+import time
+
+def getDataFrame(path):
+    data = pd.read_csv(path, on_bad_lines='skip', header=0, names=["text"])
+    return pl.from_pandas(data)
+
+def save_to_csv(dataframe):
+    folder_path = "data"
+    if not dataframe.is_empty():
+        os.makedirs(folder_path, exist_ok=True)
+        csv_path = f"{folder_path}/{uuid.uuid4()}.csv"
+        dataframe.write_csv(csv_path, separator="\t")
+        return csv_path
+
+def delete_folder_periodically(path, interval=3600):
+    while True:
+        if os.path.exists(path):
+            shutil.rmtree(path)
+        os.makedirs(path, exist_ok=True)
+        time.sleep(interval)