Spaces:

TdelaSelle
/

Paraphrasis

Running on Zero

App Files Files Community

TdelaSelle commited on Jun 17

Commit

da96d8d

1 Parent(s): 141e24d

Add semtantic segmentation + Change layout + minor fixes + drop hide negatives

Browse files

Files changed (4) hide show

Paraphrasis_main.html +304 -0
app.py +262 -48
paraphrasis_annotator.html +1 -71
requirements.txt +2 -1

Paraphrasis_main.html ADDED Viewed

	@@ -0,0 +1,304 @@

+<!DOCTYPE html>
+<html lang="fr">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Paraphrasis — Similarité sémantique entre corpus</title>
+<meta name="description" content="Paraphrasis projette deux corpus de textes dans un espace vectoriel dense et mesure leur proximité sémantique phrase à phrase, par similarité cosinus. Outil de linguistique computationnelle pour les sciences humaines.">
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+<link href="https://fonts.googleapis.com/css2?family=Fraunces:ital,opsz,wght@0,9..144,400;0,9..144,600;1,9..144,400&family=Spectral:ital,wght@0,400;0,500;1,400&family=IBM+Plex+Mono:wght@400;500&display=swap" rel="stylesheet">
+<style>
+  :root{
+    --paper:    #e7e3d9;
+    --paper-2:  #ddd8cb;
+    --ink:      #1b1d21;
+    --ink-soft: #6c685d;
+    --lapis:    #28349a;
+    --lapis-ink:#1d2670;
+    --hair:     #c7c2b4;
+    --maxw: 1140px;
+  }
+  *{box-sizing:border-box;}
+  html{-webkit-text-size-adjust:100%;}
+  body{
+    margin:0;
+    background:var(--paper);
+    color:var(--ink);
+    font-family:"Spectral",Georgia,serif;
+    font-size:18px;
+    line-height:1.65;
+    -webkit-font-smoothing:antialiased;
+  }
+  ::selection{background:var(--lapis);color:var(--paper);}
+  a{color:var(--lapis-ink);text-decoration-thickness:1px;text-underline-offset:3px;}
+  a:focus-visible,button:focus-visible{
+    outline:2px solid var(--lapis);
+    outline-offset:3px;
+  }
+  .wrap{max-width:var(--maxw);margin:0 auto;padding:0 28px;}
+  .mono{
+    font-family:"IBM Plex Mono",ui-monospace,monospace;
+    font-size:.72rem;
+    letter-spacing:.14em;
+    text-transform:uppercase;
+  }
+  /* ---------- Hero ---------- */
+  .hero{padding:64px 0 40px;}
+  .hero-grid{
+    display:grid;
+    grid-template-columns:1.15fr .85fr;
+    gap:48px;
+    align-items:center;
+  }
+  .eyebrow{color:var(--lapis-ink);margin:0 0 22px;}
+  .eyebrow::before{
+    content:"";display:inline-block;width:26px;height:1px;
+    background:var(--lapis-ink);vertical-align:middle;margin-right:12px;
+  }
+  h1.title{
+    font-family:"Fraunces",serif;
+    font-weight:600;
+    font-size:clamp(3.4rem,9vw,6.6rem);
+    line-height:.92;
+    letter-spacing:-.015em;
+    margin:0;
+  }
+  h1.title .dot{color:var(--lapis);}
+  .thesis{
+    font-size:clamp(1.15rem,2.4vw,1.5rem);
+    line-height:1.4;
+    max-width:30ch;
+    margin:24px 0 0;
+    color:var(--ink);
+  }
+  .thesis em{font-style:italic;color:var(--lapis-ink);}
+  /* ---------- Signature: cosine diagram ---------- */
+  .figure{align-self:center;}
+  .figure svg{width:100%;height:auto;display:block;}
+  .figure figcaption{
+    margin-top:14px;text-align:center;color:var(--ink-soft);
+  }
+  @media (prefers-reduced-motion:no-preference){
+    .vec{stroke-dasharray:230;stroke-dashoffset:230;animation:draw 1s cubic-bezier(.2,.7,.2,1) .25s forwards;}
+    .vec-b{animation-delay:.5s;}
+    .arc{stroke-dasharray:90;stroke-dashoffset:90;animation:draw .7s ease .95s forwards;}
+    .lbl{opacity:0;animation:fade .5s ease 1.2s forwards;}
+    .tip{opacity:0;animation:fade .4s ease forwards;}
+    .tip-a{animation-delay:1.05s;}
+    .tip-b{animation-delay:1.25s;}
+  }
+  @keyframes draw{to{stroke-dashoffset:0;}}
+  @keyframes fade{to{opacity:1;}}
+  /* ---------- Method ---------- */
+  .method{
+    border-top:1px solid var(--hair);
+    padding:54px 0 30px;
+  }
+  .intro{
+    font-size:1.18rem;
+    max-width:62ch;
+    margin:0 0 52px;
+  }
+  .steps{
+    display:grid;
+    grid-template-columns:repeat(3,1fr);
+    gap:40px;
+    counter-reset:step;
+  }
+  .step{padding-top:18px;border-top:2px solid var(--ink);}
+  .step .num{color:var(--lapis-ink);display:block;margin-bottom:14px;}
+  .step h3{
+    font-family:"Fraunces",serif;
+    font-weight:600;
+    font-size:1.45rem;
+    margin:0 0 8px;
+    letter-spacing:-.01em;
+  }
+  .step p{margin:0;font-size:1rem;color:var(--ink);}
+  .step p code{
+    font-family:"IBM Plex Mono",monospace;
+    font-size:.9em;
+    background:var(--paper-2);
+    padding:1px 5px;border-radius:3px;
+  }
+  /* ---------- App ---------- */
+  .app{padding:46px 0 20px;}
+  .app-head{
+    display:flex;justify-content:space-between;align-items:baseline;
+    gap:20px;flex-wrap:wrap;margin-bottom:18px;
+  }
+  .app-head h2{
+    font-family:"Fraunces",serif;font-weight:600;
+    font-size:1.6rem;margin:0;letter-spacing:-.01em;
+  }
+  .fullscreen{
+    font-family:"IBM Plex Mono",monospace;
+    font-size:.78rem;letter-spacing:.04em;
+    text-transform:uppercase;white-space:nowrap;
+  }
+  .note{
+    font-size:.95rem;color:var(--ink-soft);
+    margin:0 0 18px;max-width:64ch;
+  }
+  .frame{
+    position:relative;
+    border:1px solid var(--hair);
+    background:var(--paper-2);
+    border-radius:2px;
+    overflow:hidden;
+  }
+  .frame::before{
+    /* placeholder shown until the iframe paints over it */
+    content:"Chargement de l’application…";
+    position:absolute;inset:0;
+    display:flex;align-items:center;justify-content:center;
+    font-family:"IBM Plex Mono",monospace;
+    font-size:.8rem;letter-spacing:.1em;text-transform:uppercase;
+    color:var(--ink-soft);
+  }
+  .frame iframe{
+    position:relative;
+    display:block;width:100%;height:900px;border:0;background:transparent;
+  }
+  /* ---------- Footer ---------- */
+  footer{
+    border-top:1px solid var(--hair);
+    margin-top:54px;padding:38px 0 60px;
+    color:var(--ink-soft);font-size:.95rem;
+  }
+  footer .foot-grid{display:grid;grid-template-columns:1.4fr 1fr;gap:32px;}
+  footer strong{color:var(--ink);font-weight:500;}
+  footer .colophon{text-align:right;}
+  /* ---------- Responsive ---------- */
+  @media (max-width:820px){
+    .hero-grid{grid-template-columns:1fr;gap:36px;}
+    .figure{max-width:360px;margin:0 auto;}
+    .steps{grid-template-columns:1fr;gap:30px;}
+    .frame iframe{height:78vh;min-height:620px;}
+    footer .foot-grid{grid-template-columns:1fr;gap:20px;}
+    footer .colophon{text-align:left;}
+  }
+</style>
+</head>
+<body>
+  <main class="wrap">
+    <!-- HERO -->
+    <section class="hero">
+      <div class="hero-grid">
+        <div>
+          <p class="eyebrow mono">Linguistique computationnelle</p>
+          <h1 class="title">Paraphrasis<span class="dot">.</span></h1>
+          <p class="thesis">La proximité sémantique entre deux corpus, mesurée <em>phrase à phrase</em>.</p>
+        </div>
+        <figure class="figure">
+          <!-- Signature: cosine similarity is the cosine of the angle between two embedding vectors -->
+          <svg viewBox="0 0 320 250" role="img" aria-label="Diagramme de la similarité cosinus : l’angle thêta entre deux vecteurs de phrases A et B.">
+            <!-- axes hint -->
+            <line x1="60" y1="225" x2="300" y2="225" stroke="var(--hair)" stroke-width="1"/>
+            <line x1="60" y1="225" x2="60" y2="20" stroke="var(--hair)" stroke-width="1"/>
+            <!-- angle arc -->
+            <path class="arc" d="M 96.8 184.1 A 55 55 0 0 1 83.2 175.2"
+                  fill="none" stroke="var(--lapis)" stroke-width="1.5"/>
+            <!-- vector A -->
+            <line class="vec vec-a" x1="60" y1="225" x2="144.5" y2="43.7"
+                  stroke="var(--ink)" stroke-width="2.5" stroke-linecap="round"/>
+            <!-- vector B -->
+            <line class="vec vec-b" x1="60" y1="225" x2="193.8" y2="76.4"
+                  stroke="var(--lapis)" stroke-width="2.5" stroke-linecap="round"/>
+            <!-- tips -->
+            <circle class="tip tip-a" cx="144.5" cy="43.7" r="4.5" fill="var(--ink)"/>
+            <circle class="tip tip-b" cx="193.8" cy="76.4" r="4.5" fill="var(--lapis)"/>
+            <!-- labels -->
+            <g class="lbl" font-family="IBM Plex Mono, monospace">
+              <text x="150" y="38" font-size="15" fill="var(--ink)">A</text>
+              <text x="201" y="72" font-size="15" fill="var(--lapis-ink)">B</text>
+              <text x="104" y="160" font-size="15" font-style="italic" fill="var(--lapis-ink)" font-family="Spectral, serif">θ</text>
+              <text x="44" y="240" font-size="12" fill="var(--ink-soft)">O</text>
+            </g>
+          </svg>
+          <figcaption class="mono">similarité = cos θ</figcaption>
+        </figure>
+      </div>
+    </section>
+    <!-- METHOD -->
+    <section class="method">
+      <p class="intro">
+        Paraphrasis projette chaque phrase de deux corpus dans un espace vectoriel dense,
+        puis quantifie leur ressemblance par la similarité cosinus. L’outil offre aux
+        chercheur·e·s en sciences humaines un moyen d’explorer correspondances, échos et
+        reformulations entre textes, à une échelle que la lecture rapprochée seule n’atteint pas.
+      </p>
+      <div class="steps">
+        <div class="step">
+          <span class="num mono">01 · Encodage</span>
+          <h3>Plongement</h3>
+          <p>Chaque phrase est encodée par un modèle de type transformeur (<em>Sentence-Transformers</em>), entraîné à rapprocher les énoncés de sens voisin.</p>
+        </div>
+        <div class="step">
+          <span class="num mono">02 · Représentation</span>
+          <h3>Espace vectoriel</h3>
+          <p>Les phrases deviennent des vecteurs denses dans <code>ℝ^d</code>, où <code>d</code> vaut 384 à 768 selon le modèle retenu.</p>
+        </div>
+        <div class="step">
+          <span class="num mono">03 · Mesure</span>
+          <h3>Cosinus</h3>
+          <p>La similarité compare l’orientation des vecteurs : <code>cos θ = 1</code> pour une équivalence sémantique, <code>0</code> pour l’indépendance.</p>
+        </div>
+      </div>
+    </section>
+    <!-- APP -->
+    <section class="app">
+      <div class="app-head">
+        <h2>L’outil</h2>
+        <a class="fullscreen" href="https://tdelaselle-paraphrasis.hf.space/" target="_blank" rel="noopener">Ouvrir en plein écran ↗</a>
+      </div>
+      <p class="note">
+        L’application est hébergée sur Hugging Face Spaces et calcule sur GPU.
+        Après une période d’inactivité, son démarrage peut demander quelques instants —
+        laissez la fenêtre se charger.
+      </p>
+      <div class="frame">
+        <iframe
+          src="https://tdelaselle-paraphrasis.hf.space/?embed=true"
+          title="Application Paraphrasis"
+          loading="lazy"
+          allow="clipboard-write"></iframe>
+      </div>
+    </section>
+    <!-- FOOTER -->
+    <footer>
+      <div class="foot-grid">
+        <div>
+          <strong>Paraphrasis</strong> — projet de linguistique computationnelle
+          appliqué aux textes culturels. Les corpus sont traités à la demande&nbsp;;
+          aucune donnée n’est conservée après le calcul.
+        </div>
+        <div class="colophon mono">
+          Plongements · Similarité cosinus<br>
+          Hébergé sur Hugging&nbsp;Face Spaces
+        </div>
+      </div>
+    </footer>
+  </main>
+</body>
+</html>

app.py CHANGED Viewed

@@ -295,7 +295,7 @@ st.markdown(
         .panel-title {
             margin-bottom: 0.35rem;
             font-family: "IBM Plex Mono", monospace;
-            font-size: 0.78rem;
             text-transform: uppercase;
             letter-spacing: 0.08em;
             color: var(--ink-mute);
@@ -469,6 +469,22 @@ st.markdown(
             background: var(--paper);
         }
         .review-banner {
             margin-bottom: 0.85rem;
             padding: 0.65rem 0.9rem;
@@ -732,11 +748,12 @@ def filter_matches_by_threshold(
     return matches
-def build_detection_cache_key(source_text, target_text, model_name, finetune=False):
     """Builds a stable cache key so threshold-only reruns reuse embeddings."""
     payload = "||".join([
         model_name.strip(),
         "finetune" if finetune else "base",
         source_text.strip(),
         target_text.strip(),
     ])
@@ -1149,7 +1166,128 @@ def resolve_detection_input(uploaded_file, raw_text):
         "error": f"Unsupported file type for {filename}. Use .txt, .csv, or .tsv.",
     }
-def segment_and_index(text):
     """Segments text into clauses and assigns a 1-based ID to each."""
     # Clean newlines and split based on punctuation
     text = text.replace('\n', ' ')
@@ -1170,8 +1308,8 @@ def segment_and_index(text):
         'enim', 'vero', 'nam', 'sicut', 'igitur', 'ergo'
     }
-    # We set strict limits: minimum 10 words, maximum 100 words per chunk
-    segmenter = ClauseSegmenter(min_words=10, max_words=100, linguistic_markers=linguistic_markers)
     raw_sentences = segmenter.segment(text)
@@ -1200,9 +1338,9 @@ with tab1:
     st.markdown(
         """
                 <div class="hero-shell">
-                    <h1 class="hero-mark">Paraphrasis detector</h1>
                     <div class="hero-row">
-                        <div class="hero-sub">Semantic Similarity Measures through Sentence Transformers Representations</div>
                         <div class="hero-credit">by T. de la Selle, CNRS</div>
                     </div>
                 </div>
@@ -1210,13 +1348,15 @@ with tab1:
         unsafe_allow_html=True,
     )
-    left_col, right_col = st.columns([2, 1.3], gap="large")
     with left_col:
         source_file = st.file_uploader(
-            "Corpus 1 Source",
-            label_visibility="visible",
             type=["txt", "csv", "tsv"],
             key="source_file_upload",
             help=(
                 "Example corpora in various languages are available on the repository: "
                 "https://huggingface.co/spaces/TdelaSelle/Paraphrasis/tree/main/test_example "
@@ -1235,6 +1375,7 @@ with tab1:
             label_visibility="collapsed",
             type=["txt", "csv", "tsv"],
             key="target_file_upload",
         )
         text2 = st.text_area(
             "Target Text (Corpus 2)",
@@ -1245,47 +1386,108 @@ with tab1:
     with right_col:
-        st.markdown('<div class="panel-title">Settings</div>', unsafe_allow_html=True)
-        model_path = st.text_input(
-            "Hugging Face Model Path",
-            value="TdelaSelle/PatriLaSE",
-            placeholder="Enter a sentence-transformers model path...",
-            help="Select a sentence-transformers model or a language model (local path or HF hub ID). \\ The default 'TdelaSelle/PatriLaSE' is dedicated to Latin Sentence Embeddings for semantic similarity. For more general paraphrase mining, test with multilingual models like 'intfloat/multilingual-e5-large-instruct' or 'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2' (faster computation). For French texts, see 'dangvantuan/sentence-camembert-base'",
-        )
-        enable_finetune = st.checkbox(
-            "Adapted Sentence Representation fine-tuning",
-            value=False,
-            help=(
-                "Before detection, fine-tune the model on your own corpora with an "
-                "unsupervised SimCSE objective (up to 6000 sentences sampled from "
-                "corpus 1, 1 epoch, LR 1e-5, batch 32, GPU with CPU fallback)."
-                "Especially recommended for specialized domains or if base model is not a sentence-transformers model."
-            ),
-        )
-        # set a scrolling menu for selecting the task definition, with a fixed height and overflow scroll
-        task_definition = st.selectbox(
-            "Select a task",
-            options=[
-                "Semantic correspondence detection: list all sentence pairs above threshold",
-                "Semantic search:find top-k best matches in target for each source sentence",
-            ],
-            help=(
-                "The default 'Semantic correspondence detection' lists all sentence pairs with similarity above the threshold and therefore treats both corpora equally. The 'Semantic search' option finds the top-k best matches in the target corpus for each source sentence."
-            ),
-        )
-        if task_definition == "Semantic search:find top-k best matches in target for each source sentence":
-            top_k = st.number_input(
-                "Top-k matches per source sentence",
                 min_value=1,
-                max_value=20,
-                value=5,
                 step=1,
-                help="When using 'Semantic search', specify how many top matches to retrieve for each source sentence. Higher values may increase runtime and review load.",
             )
-        else: threshold = st.slider("Similarity Threshold", 0.0, 1.0, 0.70, 0.01, help="Adjust the semantic correspondence detection sensitivity. Higher values mean more similar.")
         run_detection = st.button("Run Detection", type="primary", use_container_width=True)
         run_status_box = st.container()
@@ -1339,6 +1541,11 @@ with tab1:
                     target_input["signature"],
                     model_path,
                     finetune=enable_finetune,
                 )
                 cached = st.session_state.get('detection_cache')
                 data1 = None
@@ -1355,9 +1562,16 @@ with tab1:
                     runtime_device = cached.get('runtime_device', 'cpu')
                     used_cached_similarity = True
                 else:
-                    update_detection_progress(0.15, "Segmenting corpora...")
-                    data1 = source_input["data_rows"] if source_input["data_rows"] is not None else segment_and_index(source_input["raw_text"])
-                    data2 = target_input["data_rows"] if target_input["data_rows"] is not None else segment_and_index(target_input["raw_text"])
                     if not data1 or not data2:
                         st.session_state['detection_feedback'] = {

         .panel-title {
             margin-bottom: 0.35rem;
             font-family: "IBM Plex Mono", monospace;
+            font-size: 1.25rem;
             text-transform: uppercase;
             letter-spacing: 0.08em;
             color: var(--ink-mute);
             background: var(--paper);
         }
+        /* Tighten vertical spacing around st.divider().
+           The space is the vertical block's flex gap, not the hr margin, so we
+           pull the divider's element container in with negative margins. */
+        [data-testid="stElementContainer"]:has(> [data-testid="stDivider"]),
+        [data-testid="stElementContainer"]:has(> hr) {
+            margin-top: -0.3rem !important;
+            margin-bottom: -0.3rem !important;
+        }
+        [data-testid="stDivider"],
+        [data-testid="stDivider"] hr,
+        hr {
+            margin-top: 0 !important;
+            margin-bottom: 0 !important;
+        }
         .review-banner {
             margin-bottom: 0.85rem;
             padding: 0.65rem 0.9rem;
     return matches
+def build_detection_cache_key(source_text, target_text, model_name, finetune=False, segmentation="clause"):
     """Builds a stable cache key so threshold-only reruns reuse embeddings."""
     payload = "||".join([
         model_name.strip(),
         "finetune" if finetune else "base",
+        f"seg:{segmentation}",
         source_text.strip(),
         target_text.strip(),
     ])
         "error": f"Unsupported file type for {filename}. Use .txt, .csv, or .tsv.",
     }
+@st.cache_resource
+def get_sat_model(model_name="sat-3l-sm"):
+    """Loads and caches the SaT (Segment any Text) sentence segmenter.
+    SaT models from wtpsplit are multilingual and language-agnostic: they do
+    not require a language code, so the language is effectively selected
+    automatically. The model is moved to GPU (half precision) when CUDA is
+    usable, with a transparent CPU fallback.
+    """
+    from wtpsplit import SaT
+    model = SaT(model_name)
+    if _select_inference_device() == "cuda":
+        try:
+            model.half().to("cuda")
+        except Exception:
+            # Keep the CPU model if moving to GPU fails on this machine.
+            pass
+    return model
+MIN_SEGMENT_WORDS = 8
+# SaT boundary-probability threshold: higher -> fewer/longer sentences.
+SAT_DEFAULT_THRESHOLD = 0.5
+# Punctuation marks that close a full sentence (as opposed to a clause break).
+_SENTENCE_END_CHARS = ('.', '!')
+def _ends_sentence(segment):
+    """True when a segment ends on sentence-closing punctuation (. or !)."""
+    return segment.rstrip().rstrip('"”’\')').rstrip().endswith(_SENTENCE_END_CHARS)
+def _merge_short_segments(segments, min_segment_length=MIN_SEGMENT_WORDS):
+    """Merges segments shorter than `min_segment_length` words into a neighbor.
+    SaT can over-segment into tiny fragments (e.g. "dicit illis iesus:"). This
+    fallback, mirroring the orphan-merge in segmenter.py, folds each too-short
+    segment into a neighbor based on surrounding sentence punctuation:
+    - current does NOT end on .!  and previous DOES   -> merge with next
+      (the fragment opens a new sentence; keep it with what follows).
+    - current DOES end on .!      and previous does NOT -> merge with previous
+      (the fragment closes the previous sentence).
+    - otherwise                                        -> merge with the
+      shorter neighbor (in words).
+    Edge fragments with only one neighbor merge into that neighbor.
+    """
+    segments = [s.strip() for s in segments if s and s.strip()]
+    if len(segments) <= 1:
+        return segments
+    def word_count(segment):
+        return len(segment.split())
+    # Restart after each merge: indices shift and a merged segment may still be
+    # short, so it gets re-evaluated on the next pass until everything is stable.
+    changed = True
+    while changed and len(segments) > 1:
+        changed = False
+        for i, segment in enumerate(segments):
+            if word_count(segment) >= min_segment_length:
+                continue
+            prev_segment = segments[i - 1] if i > 0 else None
+            next_segment = segments[i + 1] if i < len(segments) - 1 else None
+            if prev_segment is None:
+                merge_with_next = True
+            elif next_segment is None:
+                merge_with_next = False
+            elif not _ends_sentence(segment) and _ends_sentence(prev_segment):
+                merge_with_next = True
+            elif _ends_sentence(segment) and not _ends_sentence(prev_segment):
+                merge_with_next = False
+            else:
+                merge_with_next = word_count(next_segment) < word_count(prev_segment)
+            if merge_with_next:
+                segments[i] = f"{segment} {segments[i + 1]}"
+                del segments[i + 1]
+            else:
+                segments[i - 1] = f"{segments[i - 1]} {segment}"
+                del segments[i]
+            changed = True
+            break
+    return segments
+def segment_and_index_sat(text, min_segment_length=MIN_SEGMENT_WORDS, threshold=None):
+    """Segments text into sentences with SaT and assigns a 1-based ID to each.
+    Uses SaT (Segment any Text, wtpsplit) for multilingual semantic sentence
+    segmentation. The language is detected automatically by the model, and the
+    text is split on sentence boundaries. Robust to missing punctuation and
+    noisy line breaks. A merge fallback folds over-short segments into a
+    neighbor (see _merge_short_segments).
+    `threshold` is the boundary probability above which SaT marks a sentence
+    split (0-1). Higher values yield fewer, longer sentences; lower values
+    split more aggressively. When None, SaT's model default is used.
+    """
+    if not text or not text.strip():
+        return []
+    model = get_sat_model()
+    split_kwargs = {} if threshold is None else {"threshold": threshold}
+    sentences = model.split(text, **split_kwargs)
+    merged = _merge_short_segments(sentences, min_segment_length=min_segment_length)
+    return [
+        {"id": i, "text": sentence.strip()}
+        for i, sentence in enumerate(merged, start=1)
+    ]
+def segment_and_index(text, min_segment_length=MIN_SEGMENT_WORDS):
     """Segments text into clauses and assigns a 1-based ID to each."""
     # Clean newlines and split based on punctuation
     text = text.replace('\n', ' ')
         'enim', 'vero', 'nam', 'sicut', 'igitur', 'ergo'
     }
+    # We set strict limits: minimum 10 words, maximum 150 words per chunk
+    segmenter = ClauseSegmenter(min_words=min_segment_length, max_words=150, linguistic_markers=linguistic_markers)
     raw_sentences = segmenter.segment(text)
     st.markdown(
         """
                 <div class="hero-shell">
+                    <h1 class="hero-mark">Paraphrasis analyzer</h1>
                     <div class="hero-row">
+                        <div class="hero-sub">Semantic Similarity Measures through Sentence Representations</div>
                         <div class="hero-credit">by T. de la Selle, CNRS</div>
                     </div>
                 </div>
         unsafe_allow_html=True,
     )
+    left_col, right_col = st.columns([1.6, 1.56], gap="large")
     with left_col:
+        st.markdown('<div class="panel-title">Corpus 1 Source</div>', unsafe_allow_html=True)
         source_file = st.file_uploader(
+            label="Corpus 1 Source",
+            label_visibility="collapsed",
             type=["txt", "csv", "tsv"],
             key="source_file_upload",
+            max_upload_size=1,
             help=(
                 "Example corpora in various languages are available on the repository: "
                 "https://huggingface.co/spaces/TdelaSelle/Paraphrasis/tree/main/test_example "
             label_visibility="collapsed",
             type=["txt", "csv", "tsv"],
             key="target_file_upload",
+            max_upload_size=1,
         )
         text2 = st.text_area(
             "Target Text (Corpus 2)",
     with right_col:
+        # --- MODEL ---
+        st.markdown('<div class="panel-title">Model</div>', unsafe_allow_html=True)
+        model_col, finetune_col = st.columns([3, 2], gap="small")
+        with model_col:
+            model_path = st.text_input(
+                "Hugging Face Model Path",
+                value="TdelaSelle/PatriSBERT-STS",
+                # label_visibility="collapsed",
+                placeholder="Enter a sentence-transformers model path...",
+                help="Select a sentence-transformers model or a language model (local path or HF hub ID). \\ The default 'TdelaSelle/PatriLaSE' is dedicated to Latin Sentence Embeddings for semantic similarity. For more general paraphrase mining, test with multilingual models like 'intfloat/multilingual-e5-large-instruct' or 'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2' (faster computation). For French texts, see 'dangvantuan/sentence-camembert-base'",
+            )
+        with finetune_col:
+            enable_finetune = st.checkbox(
+                "Model fine-tuning on corpus 1",
+                value=False,
+                help=(
+                    "Before detection, fine-tune the model on your own corpora with an "
+                    "unsupervised SimCSE objective (up to 6000 sentences sampled from "
+                    "corpus 1, 1 epoch, LR 1e-5, batch 32, GPU with CPU fallback)."
+                    "Use it if base model is not a sentence-transformers model or if data is domain-specific or noisy. Fine-tuning is not needed for general-purpose sentence-transformers models."
+                ),
+            )
+        st.divider()
+        # --- SEGMENTATION ---
+        st.markdown('<div class="panel-title">Segmentation</div>', unsafe_allow_html=True)
+        min_words_col, sat_col = st.columns([1,3], gap="small")
+        with min_words_col:
+            min_segment_words = st.number_input(
+                "Minimum words per segment",
                 min_value=1,
+                max_value=50,
+                value=MIN_SEGMENT_WORDS,
                 step=1,
+                help=(
+                    "Shortest segment kept during segmentation. With the clause "
+                    "segmenter this is the minimum chunk size; with SaT it drives the "
+                    "merge fallback that folds shorter sentences into a neighbor. "
+                    "Applies only to raw text inputs."
+                ),
+            )
+        with sat_col:
+            use_sat_segmentation = st.checkbox(
+                "Semantic segmentation (splitting into sentences even without/noisy punctuation)",
+                value=False,
+                help=(
+                    "Segment pasted/.txt corpora into sentences with SaT (wtpsplit) "
+                    "instead of the default clause segmenter. SaT is multilingual and "
+                    "selects the language automatically, splitting on sentence "
+                    "boundaries even when punctuation is missing. Applies only to raw "
+                    "text inputs; pre-segmented .csv/.tsv corpora are used as provided."
+                ),
             )
+        sat_threshold = SAT_DEFAULT_THRESHOLD
+        if use_sat_segmentation:
+            sat_threshold = st.slider(
+                "SaT segmentation threshold",
+                min_value=0.05,
+                max_value=0.95,
+                value=SAT_DEFAULT_THRESHOLD,
+                step=0.05,
+                format="%.2f",
+                help=(
+                    "Boundary probability above which SaT marks a sentence split. "
+                    "Higher values produce fewer, longer sentences; lower values "
+                    "split more aggressively. Default ~0.025."
+                ),
+            )
+        st.divider()
+        # --- TASK ---
+        st.markdown('<div class="panel-title">Task</div>', unsafe_allow_html=True)
+        task_col, param_col = st.columns([5, 2], gap="small")
+        with task_col:
+            # set a scrolling menu for selecting the task definition, with a fixed height and overflow scroll
+            task_definition = st.selectbox(
+                "Select a task",
+                options=[
+                    "Semantic correspondence detection: list all sentence pairs above threshold",
+                    "Semantic search:find top-k best matches in target for each source sentence",
+                ],
+                help=(
+                    "The default 'Semantic correspondence detection' lists all sentence pairs with similarity above the threshold and therefore treats both corpora equally. The 'Semantic search' option finds the top-k best matches in the target corpus for each source sentence."
+                ),
+            )
+        with param_col:
+            if task_definition == "Semantic search:find top-k best matches in target for each source sentence":
+                top_k = st.number_input(
+                    "Top-k matches per source sentence",
+                    min_value=1,
+                    max_value=20,
+                    value=5,
+                    step=1,
+                    help="When using 'Semantic search', specify how many top matches to retrieve for each source sentence. Higher values may increase runtime and review load.",
+                )
+            else:
+                threshold = st.slider("Similarity Threshold", 0.0, 1.0, 0.80, 0.01, help="Adjust the semantic correspondence detection sensitivity. Higher values mean more similar.")
+        # st.divider()
         run_detection = st.button("Run Detection", type="primary", use_container_width=True)
         run_status_box = st.container()
                     target_input["signature"],
                     model_path,
                     finetune=enable_finetune,
+                    segmentation=(
+                        f"sat:{sat_threshold:.3f}:min{min_segment_words}"
+                        if use_sat_segmentation
+                        else f"clause:min{min_segment_words}"
+                    ),
                 )
                 cached = st.session_state.get('detection_cache')
                 data1 = None
                     runtime_device = cached.get('runtime_device', 'cpu')
                     used_cached_similarity = True
                 else:
+                    segment_text = segment_and_index_sat if use_sat_segmentation else segment_and_index
+                    segment_kwargs = (
+                        {"min_segment_length": min_segment_words, "threshold": sat_threshold}
+                        if use_sat_segmentation
+                        else {"min_segment_length": min_segment_words}
+                    )
+                    segmentation_label = "SaT (Segment any Text)" if use_sat_segmentation else "clause"
+                    update_detection_progress(0.15, f"Segmenting corpora ({segmentation_label})...")
+                    data1 = source_input["data_rows"] if source_input["data_rows"] is not None else segment_text(source_input["raw_text"], **segment_kwargs)
+                    data2 = target_input["data_rows"] if target_input["data_rows"] is not None else segment_text(target_input["raw_text"], **segment_kwargs)
                     if not data1 or not data2:
                         st.session_state['detection_feedback'] = {

paraphrasis_annotator.html CHANGED Viewed

@@ -721,10 +721,6 @@
         <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 6h12"/><path d="M3 12h9"/><path d="M3 18h6"/><path d="M17 4v16"/><polyline points="13 16 17 20 21 16"/></svg>
         <span id="btnSortLabel">Similarity</span>
       </button>
-      <button class="btn btn-toggle" id="btnHideNeg" title="Hide pairs labelled as rejected">
-        <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M17.94 17.94A10.07 10.07 0 0112 20c-7 0-11-8-11-8a18.45 18.45 0 015.06-5.94M9.9 4.24A9.12 9.12 0 0112 4c7 0 11 8 11 8a18.5 18.5 0 01-2.16 3.19m-6.72-1.07a3 3 0 11-4.24-4.24"/><line x1="1" y1="1" x2="23" y2="23"/></svg>
-        Hide&nbsp;negatives
-      </button>
     </div>
     <div class="ctrl-group">
@@ -831,7 +827,6 @@ const state = {
   threshold: 0.95,
   filterText: '',
   sortMode: 'similarity',   // 'similarity' (file order) | 'id' (by Source_ID)
-  hideNegatives: false,     // hide pairs tagged as rejected
   pairsFileName: null,
   c1FileName: null,
   c2FileName: null,
@@ -1081,7 +1076,6 @@ function applyFilter() {
   state.filtered = [];
   for (let i = 0; i < state.pairs.length; i++) {
     const p = state.pairs[i];
-    if (state.hideNegatives && p.tag === 'r') continue;
     if (t && !(String(p.idA).toLowerCase().includes(t) || String(p.idB).toLowerCase().includes(t))) continue;
     state.filtered.push(i);
   }
@@ -1252,34 +1246,6 @@ function applyTag(tag) {
   });
   updateStats();
-  // If Hide-negatives is on and we just rejected rows, they must disappear
-  // from the filtered view. Remember the *next* visible pair's identity before
-  // re-filtering so we can land on it, not on the first row in the list.
-  if (state.hideNegatives && tag === 'r') {
-    const last = indices[indices.length - 1];
-    let anchorPairIdx = -1;
-    for (let fi = last + 1; fi < state.filtered.length; fi++) {
-      const pi = state.filtered[fi];
-      if (state.pairs[pi].tag !== 'r') { anchorPairIdx = pi; break; }
-    }
-    applyFilter();
-    state.selection.clear();
-    if (anchorPairIdx >= 0) {
-      const next = state.filtered.indexOf(anchorPairIdx);
-      if (next >= 0) { selectOne(next); return; }
-    }
-    // No trailing visible row → fall back to whatever is still in view.
-    if (state.filtered.length > 0) {
-      const target = Math.min(indices[0], state.filtered.length - 1);
-      selectOne(target);
-    } else {
-      state.pivot = -1;
-      renderVisibleRows();
-      renderCurrentTexts();
-    }
-    return;
-  }
   // Normal auto-advance to the next row.
   const last = indices[indices.length - 1];
   const next = last + 1;
@@ -1445,7 +1411,7 @@ function readFile(file, cb) {
 }
 /* =========================================================
-   TOGGLES: SORT & HIDE-NEGATIVES
    ========================================================= */
 function toggleSort() {
   const pivotPairIdx = state.pivot >= 0 ? state.filtered[state.pivot] : -1;
@@ -1472,50 +1438,15 @@ function toggleSort() {
     : 'Restored similarity order.');
 }
-function toggleHideNegatives() {
-  const pivotPairIdx = state.pivot >= 0 ? state.filtered[state.pivot] : -1;
-  const selPairs = new Set([...state.selection].map(fi => state.filtered[fi]));
-  state.hideNegatives = !state.hideNegatives;
-  applyFilter();
-  // Remap selection and pivot after filter change.
-  state.selection.clear();
-  for (let i = 0; i < state.filtered.length; i++) {
-    if (selPairs.has(state.filtered[i])) state.selection.add(i);
-  }
-  if (pivotPairIdx >= 0) {
-    const newPivot = state.filtered.indexOf(pivotPairIdx);
-    if (newPivot >= 0) {
-      state.pivot = newPivot;
-      scrollFilteredIntoView(newPivot);
-    } else if (state.filtered.length > 0) {
-      // Pivot was itself a negative that just got hidden — fall back to row 0.
-      selectOne(0);
-    } else {
-      state.pivot = -1;
-      renderCurrentTexts();
-    }
-  }
-  renderVisibleRows();
-  updateToggleButtons();
-  const nHidden = state.pairs.filter(p => p.tag === 'r').length;
-  toast(state.hideNegatives
-    ? `Hiding ${nHidden.toLocaleString()} rejected pair${nHidden === 1 ? '' : 's'}.`
-    : 'Showing all pairs.');
-}
 function updateToggleButtons() {
   const bS = document.getElementById('btnSort');
   const bSLabel = document.getElementById('btnSortLabel');
-  const bH = document.getElementById('btnHideNeg');
   if (bS && bSLabel) {
     // The label always reflects the *current* sort rule.
     bSLabel.textContent = state.sortMode === 'id' ? 'Id' : 'Similarity';
     // Visually mark "Id" as the accent (non-default) mode.
     bS.classList.toggle('active', state.sortMode === 'id');
   }
-  if (bH) bH.classList.toggle('active', state.hideNegatives);
 }
@@ -1536,7 +1467,6 @@ document.getElementById('fileC2').addEventListener('change', e => {
 });
 document.getElementById('btnExport').addEventListener('click', exportPairs);
 document.getElementById('btnSort').addEventListener('click', toggleSort);
-document.getElementById('btnHideNeg').addEventListener('click', toggleHideNegatives);
 document.getElementById('pairsScroll').addEventListener('scroll', () => {
   requestAnimationFrame(renderVisibleRows);

         <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 6h12"/><path d="M3 12h9"/><path d="M3 18h6"/><path d="M17 4v16"/><polyline points="13 16 17 20 21 16"/></svg>
         <span id="btnSortLabel">Similarity</span>
       </button>
     </div>
     <div class="ctrl-group">
   threshold: 0.95,
   filterText: '',
   sortMode: 'similarity',   // 'similarity' (file order) | 'id' (by Source_ID)
   pairsFileName: null,
   c1FileName: null,
   c2FileName: null,
   state.filtered = [];
   for (let i = 0; i < state.pairs.length; i++) {
     const p = state.pairs[i];
     if (t && !(String(p.idA).toLowerCase().includes(t) || String(p.idB).toLowerCase().includes(t))) continue;
     state.filtered.push(i);
   }
   });
   updateStats();
   // Normal auto-advance to the next row.
   const last = indices[indices.length - 1];
   const next = last + 1;
 }
 /* =========================================================
+   TOGGLES: SORT
    ========================================================= */
 function toggleSort() {
   const pivotPairIdx = state.pivot >= 0 ? state.filtered[state.pivot] : -1;
     : 'Restored similarity order.');
 }
 function updateToggleButtons() {
   const bS = document.getElementById('btnSort');
   const bSLabel = document.getElementById('btnSortLabel');
   if (bS && bSLabel) {
     // The label always reflects the *current* sort rule.
     bSLabel.textContent = state.sortMode === 'id' ? 'Id' : 'Similarity';
     // Visually mark "Id" as the accent (non-default) mode.
     bS.classList.toggle('active', state.sortMode === 'id');
   }
 }
 });
 document.getElementById('btnExport').addEventListener('click', exportPairs);
 document.getElementById('btnSort').addEventListener('click', toggleSort);
 document.getElementById('pairsScroll').addEventListener('scroll', () => {
   requestAnimationFrame(renderVisibleRows);

requirements.txt CHANGED Viewed

@@ -3,4 +3,5 @@ sentence-transformers
 datasets
 accelerate>=0.26.0
 torch
-pandas

 datasets
 accelerate>=0.26.0
 torch
+pandas
+wtpsplit