initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,99 @@
+# Python build
+.eggs/
+gradio.egg-info
+dist/
+dist-lite/
+*.pyc
+__pycache__/
+*.py[cod]
+*$py.class
+build/
+!js/build/
+!js/build/dist/
+__tmp/*
+*.pyi
+!gradio/stubs/**/*.pyi
+py.typed
+.ipynb_checkpoints/
+.python-version
+=23.2
+
+# JS build
+gradio/templates/*
+gradio/node/*
+gradio/_frontend_code/*
+js/gradio-preview/test/*
+
+# Secrets
+.env
+
+# Gradio run artifacts
+*.db
+*.sqlite3
+gradio/launches.json
+gradio/hash_seed.txt
+.gradio/
+
+tmp.zip
+
+# Tests
+.coverage
+coverage.xml
+test.txt
+**/snapshots/**/*.png
+playwright-report/
+.hypothesis
+.lite-perf.json
+
+# Demos
+demo/tmp.zip
+demo/files/*.avi
+demo/files/*.mp4
+demo/all_demos/demos/*
+demo/all_demos/requirements.txt
+demo/*/config.json
+demo/annotatedimage_component/*.png
+demo/fake_diffusion_with_gif/*.gif
+demo/cancel_events/cancel_events_output_log.txt
+demo/unload_event_test/output_log.txt
+demo/stream_video_out/output_*.ts
+demo/stream_video_out/output_*.mp4
+demo/stream_audio_out/*.mp3
+
+# Etc
+.idea/*
+.DS_Store
+*.bak
+workspace.code-workspace
+*.h5
+
+# dev containers
+.pnpm-store/
+
+# log files
+.pnpm-debug.log
+
+# Local virtualenv for devs
+.venv*
+
+# FRP
+gradio/frpc_*
+.vercel
+
+# js
+node_modules
+public/build/
+test-results
+client/js/dist/*
+client/js/test.js
+.config/test.py
+.svelte-kit
+
+
+# storybook
+storybook-static
+build-storybook.log
+js/storybook/theme.css
+
+# playwright
+.config/playwright/.cache

README.md

@@ -0,0 +1,12 @@
+---
+title: Sentence Transformers
+emoji: 🏢
+colorFrom: green
+colorTo: gray
+sdk: gradio
+sdk_version: 5.3.0
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+import gradio as gr
+import pandas as pd
+import polars as pl
+from datasets import Dataset
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.util import paraphrase_mining
+import torch
+
+
+def upload_file(filepath):
+    name = Path(filepath).name
+    return [gr.UploadButton(visible=False), gr.DownloadButton(label=f"Download {name}", value=filepath, visible=True)]
+
+
+def getData(path):
+    #data = Dataset.from_csv(path, column_names=["text"])
+    data = Dataset.from_pandas(pd.read_csv(path, on_bad_lines='skip', names=["text"]))
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2",
+                                backend="openvino",
+                                device=device,
+                                trust_remote_code=True)
+
+    paraphrases = paraphrase_mining(
+        model,
+        data["text"],
+        corpus_chunk_size=len(data),
+        show_progress_bar=True,
+        batch_size=1024,
+        max_pairs=len(data) ** 2
+    )
+
+    df_pd = pd.DataFrame(paraphrases)
+    df = pl.from_pandas(df_pd)
+    df = df.rename({"0": "score", "1": "sentence_1", "2": "sentence_2"})
+
+    union_df = pl.DataFrame(data.to_pandas())
+
+    df = df.with_columns([
+        pl.col("score").round(3).cast(pl.Float32),
+        union_df.select(pl.col("text")).to_series()[df["sentence_1"].cast(pl.Int32)].alias("sentence_1"),
+        union_df.select(pl.col("text")).to_series()[df["sentence_2"].cast(pl.Int32)].alias("sentence_2"),
+    ]).filter(pl.col("score") > 0.96).sort(["score"], descending=True)
+
+    data = pl.from_arrow(data.data.table)
+
+    return [data, df]
+
+
+with gr.Blocks() as demo:
+    with gr.Column():
+        upload_button = gr.UploadButton(label="upload csv", file_types=['.csv'], file_count="single")
+        output_data = gr.Dataframe(headers=["text"], col_count=1, label="Uploaded Data")
+        output_paraphrases = gr.Dataframe(headers=["score", "sentence_1", "sentence_2"], type="polars",
+                                          label="Paraphrase Mining Results")
+
+        upload_button.upload(fn=getData, inputs=upload_button, outputs=[output_data, output_paraphrases])
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,6 @@
+transformers
+torch
+pandas
+polars
+datasets
+sentence-transformers[openvino,onnx-gpu,onnx]