Upload app.py with huggingface_hub

app.py

@@ -1,493 +1,162 @@
 """
 Gradio app to explore pancreas cancer clinical report annotations.
-Loads data from rntc/biomed-fr-pancreas-annotations on HuggingFace.
 """
 
 import gradio as gr
 from datasets import load_dataset
-from difflib import SequenceMatcher
 
 # Load the dataset
-print("Loading dataset from HuggingFace...")
+print("Loading dataset...")
 dataset = load_dataset("rntc/biomed-fr-pancreas-annotations", split="train")
 print(f"Loaded {len(dataset)} samples")
 
-
-def fuzzy_find_span(text: str, span: str, threshold: float = 0.85) -> tuple:
-    """
-    Find a span in text with fuzzy matching.
-    Returns (start, end) or None if not found.
-    """
-    # First try exact match
-    idx = text.find(span)
-    if idx != -1:
-        return (idx, idx + len(span))
-
-    # Try fuzzy match with sliding window
-    span_len = len(span)
-    if span_len < 10 or span_len > len(text):
-        return None
-
-    best_ratio = 0
-    best_pos = None
-
-    # Use a window slightly larger than span
-    window_size = min(span_len + 20, len(text))
-
-    for i in range(0, len(text) - span_len + 1, max(1, span_len // 4)):
-        window = text[i:i + window_size]
-        ratio = SequenceMatcher(None, span, window[:span_len]).ratio()
-        if ratio > best_ratio and ratio >= threshold:
-            best_ratio = ratio
-            best_pos = i
-
-    if best_pos is not None:
-        return (best_pos, best_pos + span_len)
-
-    return None
+# Colors for highlighting
+COLORS = [
+    "#FFEB3B", "#4CAF50", "#2196F3", "#FF9800", "#E91E63",
+    "#9C27B0", "#00BCD4", "#8BC34A", "#FF5722", "#607D8B",
+]
 
 
-def escape_html(text: str) -> str:
-    """Escape HTML special characters."""
+def escape_html(text):
     if not text:
         return ""
-    return (str(text)
-            .replace("&", "&amp;")
-            .replace("<", "&lt;")
-            .replace(">", "&gt;")
-            .replace('"', "&quot;"))
-
-
-# Soft pastel colors for better readability
-COLORS = [
-    "#FFE082",  # amber
-    "#A5D6A7",  # green
-    "#90CAF9",  # blue
-    "#FFAB91",  # deep orange
-    "#CE93D8",  # purple
-    "#80DEEA",  # cyan
-    "#C5E1A5",  # light green
-    "#FFCC80",  # orange
-    "#B39DDB",  # deep purple
-    "#81D4FA",  # light blue
-    "#EF9A9A",  # red
-    "#FFF59D",  # yellow
-    "#F48FB1",  # pink
-    "#80CBC4",  # teal
-    "#BCAAA4",  # brown
-]
+    return str(text).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
 
 
-def highlight_spans_in_text(cr_text: str, annotation: dict) -> str:
-    """
-    Highlight spans in the CR text based on annotations.
-    Returns HTML with highlighted spans.
-    """
+def highlight_text(cr_text, annotation):
+    """Highlight spans in CR text."""
     if not cr_text or not annotation:
-        return f"<div class='cr-text'>{escape_html(cr_text)}</div>"
+        return f"<pre style='white-space:pre-wrap;'>{escape_html(cr_text)}</pre>"
 
-    # Collect all spans with their variable names
-    spans_to_highlight = []
+    # Collect valid spans (that exist in text)
+    spans = []
     for var_name, var_data in annotation.items():
         if var_data and isinstance(var_data, dict):
             span = var_data.get("span")
             value = var_data.get("value")
-            if span and value and len(span) >= 5:  # Skip very short spans
-                spans_to_highlight.append({
-                    "span": span,
-                    "var_name": var_name,
+            if span and value and span in cr_text:
+                spans.append({
+                    "text": span,
+                    "start": cr_text.find(span),
+                    "var": var_name.replace("_", " ").title(),
                     "value": str(value)
                 })
 
-    if not spans_to_highlight:
-        return f"<div class='cr-text'>{escape_html(cr_text)}</div>"
-
-    # Sort spans by length (longest first) to prioritize longer matches
-    spans_to_highlight.sort(key=lambda x: len(x["span"]), reverse=True)
-
-    # Find spans in text (with fuzzy matching)
-    found_spans = []
-    for item in spans_to_highlight:
-        result = fuzzy_find_span(cr_text, item["span"])
-        if result:
-            start, end = result
-            found_spans.append({
-                "start": start,
-                "end": end,
-                "var_name": item["var_name"],
-                "value": item["value"],
-                "span": cr_text[start:end]  # Use actual text from CR
-            })
-
-    if not found_spans:
-        return f"<div class='cr-text'>{escape_html(cr_text)}</div>"
-
-    # Sort by start position
-    found_spans.sort(key=lambda x: x["start"])
-
-    # Remove overlapping spans (keep the first/longest one)
-    non_overlapping = []
-    for span in found_spans:
-        if not non_overlapping:
-            non_overlapping.append(span)
-        elif span["start"] >= non_overlapping[-1]["end"]:
-            non_overlapping.append(span)
-
-    # Assign colors to variable names
-    var_colors = {}
+    if not spans:
+        return f"<pre style='white-space:pre-wrap;'>{escape_html(cr_text)}</pre>"
+
+    # Sort by position and remove overlaps
+    spans.sort(key=lambda x: x["start"])
+    filtered = []
+    for s in spans:
+        s["end"] = s["start"] + len(s["text"])
+        if not filtered or s["start"] >= filtered[-1]["end"]:
+            filtered.append(s)
+
+    # Build HTML
+    html = []
+    pos = 0
+    color_map = {}
     color_idx = 0
-    for span in non_overlapping:
-        if span["var_name"] not in var_colors:
-            var_colors[span["var_name"]] = COLORS[color_idx % len(COLORS)]
+
+    for s in filtered:
+        # Text before span
+        if s["start"] > pos:
+            html.append(escape_html(cr_text[pos:s["start"]]))
+
+        # Assign color
+        if s["var"] not in color_map:
+            color_map[s["var"]] = COLORS[color_idx % len(COLORS)]
             color_idx += 1
 
-    # Build HTML with highlights
-    html_parts = []
-    last_end = 0
-
-    for span in non_overlapping:
-        # Add text before this span
-        if span["start"] > last_end:
-            html_parts.append(escape_html(cr_text[last_end:span["start"]]))
-
-        # Add highlighted span
-        color = var_colors[span["var_name"]]
-        var_label = span["var_name"].replace("_", " ").replace("  ", " ").title()
-        tooltip = f"{var_label}\\n→ {span['value']}"
-
-        html_parts.append(
-            f'<mark class="entity" style="background-color: {color};" '
-            f'title="{escape_html(tooltip)}" '
-            f'data-var="{escape_html(var_label)}">'
-            f'{escape_html(span["span"])}'
-            f'<span class="entity-label">{escape_html(var_label[:20])}</span>'
-            f'</mark>'
+        # Highlighted span
+        color = color_map[s["var"]]
+        html.append(
+            f'<mark style="background:{color};padding:1px 3px;border-radius:3px;" '
+            f'title="{escape_html(s["var"])}: {escape_html(s["value"])}">'
+            f'{escape_html(s["text"])}</mark>'
         )
-        last_end = span["end"]
+        pos = s["end"]
 
-    # Add remaining text
-    if last_end < len(cr_text):
-        html_parts.append(escape_html(cr_text[last_end:]))
+    # Remaining text
+    if pos < len(cr_text):
+        html.append(escape_html(cr_text[pos:]))
 
-    html = "".join(html_parts)
-    return f"<div class='cr-text'>{html}</div>"
+    return f"<pre style='white-space:pre-wrap;line-height:1.6;'>{' '.join(html)}</pre>"
 
 
-def format_annotations_table(annotation: dict) -> str:
-    """Format annotations as an HTML table with categories."""
+def format_table(annotation):
+    """Format annotations as HTML table."""
     if not annotation:
         return "<p>No annotations</p>"
 
-    # Group variables by category (simple heuristic based on name)
-    categories = {
-        "Patient Info": ["date_of_birth", "age_at_cancer_diagnosis", "biological_gender", "vital_status", "date_of_death"],
-        "Diagnosis": ["date_of_cancer_diagnostic", "primary_tumor_localisation", "ctnm_stage", "stage_as_per_ehr", "histological_type", "epithelial_tumor_subtype"],
-        "Tumor Characteristics": ["resectability_status", "two_largest_diameters", "metastasis_localisation", "number_of_metastatic_sites"],
-        "Lab Results": ["crp_at_diagnosis", "albumin_at_diagnosis", "alanine_transaminase", "aspartate_aminotransferase", "conjugated_bilirubin", "ca19_9"],
-        "Treatment": ["surgery", "loco_regional_radiotherapy", "immunotherapy", "targeted_therapy", "full_course_of_initial_treatment"],
-        "Molecular": ["germline_mutation", "tumor_molecular_profiling"],
-        "Progression": ["date_of_first_progression", "type_of_first_progression", "treatment_at_first_progression", "best_response", "reason_for_treatment_end"],
-    }
-
-    def get_category(var_name):
-        for cat, keywords in categories.items():
-            for kw in keywords:
-                if kw in var_name.lower():
-                    return cat
-        return "Other"
-
-    # Group rows by category
-    categorized = {}
+    rows = []
     for var_name, var_data in annotation.items():
         if var_data and isinstance(var_data, dict):
             value = var_data.get("value")
-            if value:
-                cat = get_category(var_name)
-                if cat not in categorized:
-                    categorized[cat] = []
-                categorized[cat].append((var_name, var_data))
-
-    if not categorized:
-        return "<p class='no-data'>No extracted values</p>"
-
-    html_parts = []
-
-    for category in ["Patient Info", "Diagnosis", "Tumor Characteristics", "Lab Results", "Treatment", "Molecular", "Progression", "Other"]:
-        if category not in categorized:
-            continue
-
-        html_parts.append(f"<div class='category'><h4>{category}</h4>")
-        html_parts.append("<table class='annotations-table'>")
-
-        for var_name, var_data in categorized[category]:
-            value = var_data.get("value", "")
             span = var_data.get("span", "")
-            var_label = var_name.replace("_", " ").title()
-
-            span_preview = span[:80] + "..." if span and len(span) > 80 else span
-
-            html_parts.append(f"""
-                <tr>
-                    <td class='var-name'>{escape_html(var_label)}</td>
-                    <td class='var-value'>{escape_html(str(value))}</td>
-                    <td class='var-span'>{escape_html(span_preview) if span_preview else '-'}</td>
-                </tr>
-            """)
-
-        html_parts.append("</table></div>")
-
-    return "".join(html_parts)
-
-
-def get_stats(annotation: dict) -> str:
-    """Get statistics about extracted values."""
-    if not annotation:
-        return "No data"
-
-    total = len(annotation)
-    extracted = sum(1 for v in annotation.values() if v and isinstance(v, dict) and v.get("value"))
 
-    return f"📊 Extracted: {extracted}/{total} variables ({100*extracted//total}%)"
-
-
-def display_sample(sample_idx: int):
-    """Display a sample from the dataset."""
-    if sample_idx < 0 or sample_idx >= len(dataset):
-        return "Invalid sample index", "<p>Invalid sample index</p>", "Invalid"
+            var_label = var_name.replace("_", " ").title()
 
-    sample = dataset[int(sample_idx)]
-    cr_text = sample.get("CR", "")
+            if value:
+                # Check if span is a "not found" explanation
+                if span and ("pas de mention" in span.lower() or "not performed" in str(value).lower()):
+                    display_value = "/"
+                    display_span = ""
+                else:
+                    display_value = str(value)
+                    display_span = span[:60] + "..." if span and len(span) > 60 else (span or "")
+            else:
+                display_value = "/"
+                display_span = ""
+
+            rows.append(f"""<tr>
+                <td style="padding:6px 10px;border-bottom:1px solid #ddd;font-weight:500;">{escape_html(var_label)}</td>
+                <td style="padding:6px 10px;border-bottom:1px solid #ddd;color:#1565C0;">{escape_html(display_value)}</td>
+                <td style="padding:6px 10px;border-bottom:1px solid #ddd;color:#666;font-size:12px;font-style:italic;">{escape_html(display_span)}</td>
+            </tr>""")
+
+    return f"""<table style="width:100%;border-collapse:collapse;font-size:13px;">
+        <thead><tr style="background:#f5f5f5;">
+            <th style="padding:8px 10px;text-align:left;border-bottom:2px solid #ddd;">Variable</th>
+            <th style="padding:8px 10px;text-align:left;border-bottom:2px solid #ddd;">Value</th>
+            <th style="padding:8px 10px;text-align:left;border-bottom:2px solid #ddd;">Source</th>
+        </tr></thead>
+        <tbody>{"".join(rows)}</tbody>
+    </table>"""
+
+
+def display_sample(idx):
+    """Display a sample."""
+    idx = int(idx)
+    if idx < 0 or idx >= len(dataset):
+        return "Invalid index", "Invalid index"
+
+    sample = dataset[idx]
+    cr = sample.get("CR", "")
     annotation = sample.get("annotation", {})
 
-    highlighted_html = highlight_spans_in_text(cr_text, annotation)
-    annotations_html = format_annotations_table(annotation)
-    stats = get_stats(annotation)
-
-    return highlighted_html, annotations_html, stats
-
-
-def search_samples(query: str):
-    """Search samples by text content."""
-    if not query or len(query) < 3:
-        # Return first 20 samples
-        return [[i, dataset[i]["CR"][:80] + "..."] for i in range(min(20, len(dataset)))]
-
-    results = []
-    query_lower = query.lower()
-    for i, sample in enumerate(dataset):
-        cr = sample.get("CR", "")
-        if query_lower in cr.lower():
-            results.append([i, cr[:80] + "..."])
-            if len(results) >= 50:
-                break
-
-    if not results:
-        return [["No results", f"No samples found containing '{query}'"]]
-
-    return results
-
-
-# Custom CSS for better styling
-custom_css = """
-.cr-text {
-    font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
-    font-size: 14px;
-    line-height: 1.8;
-    padding: 20px;
-    background: #fafafa;
-    border-radius: 8px;
-    white-space: pre-wrap;
-    max-height: 500px;
-    overflow-y: auto;
-}
-
-.entity {
-    padding: 2px 6px;
-    border-radius: 4px;
-    cursor: help;
-    position: relative;
-    display: inline;
-    transition: all 0.2s;
-}
-
-.entity:hover {
-    filter: brightness(0.9);
-    box-shadow: 0 2px 8px rgba(0,0,0,0.15);
-}
-
-.entity-label {
-    display: none;
-    position: absolute;
-    bottom: 100%;
-    left: 0;
-    background: #333;
-    color: white;
-    padding: 4px 8px;
-    border-radius: 4px;
-    font-size: 11px;
-    white-space: nowrap;
-    z-index: 100;
-}
-
-.entity:hover .entity-label {
-    display: block;
-}
-
-.category {
-    margin-bottom: 20px;
-}
-
-.category h4 {
-    color: #1976d2;
-    border-bottom: 2px solid #1976d2;
-    padding-bottom: 8px;
-    margin-bottom: 12px;
-}
-
-.annotations-table {
-    width: 100%;
-    border-collapse: collapse;
-    font-size: 13px;
-}
-
-.annotations-table tr:nth-child(even) {
-    background: #f5f5f5;
-}
-
-.annotations-table td {
-    padding: 10px 12px;
-    border-bottom: 1px solid #e0e0e0;
-    vertical-align: top;
-}
-
-.var-name {
-    font-weight: 600;
-    color: #333;
-    width: 30%;
-}
-
-.var-value {
-    color: #1976d2;
-    font-weight: 500;
-    width: 25%;
-}
-
-.var-span {
-    color: #666;
-    font-style: italic;
-    font-size: 12px;
-    width: 45%;
-}
-
-.no-data {
-    color: #999;
-    font-style: italic;
-    padding: 20px;
-    text-align: center;
-}
-
-.stats-badge {
-    background: #e3f2fd;
-    color: #1976d2;
-    padding: 8px 16px;
-    border-radius: 20px;
-    font-weight: 500;
-    display: inline-block;
-}
-"""
+    return highlight_text(cr, annotation), format_table(annotation)
 
 
-# Build the Gradio interface
-with gr.Blocks(
-    title="Pancreas Cancer Annotations Explorer",
-    theme=gr.themes.Soft(primary_hue="blue"),
-    css=custom_css
-) as demo:
-
-    gr.Markdown("""
-    # 🔬 Pancreas Cancer Clinical Report Annotations Explorer
-
-    Explore structured annotations extracted from synthetic French clinical reports about pancreas cancer.
-
-    **How to use:**
-    - Use the slider or search to navigate samples
-    - Hover over highlighted text to see extracted variables
-    - View the complete annotation table below
-    """)
+# Build UI
+with gr.Blocks(title="Pancreas Annotations", theme=gr.themes.Base()) as demo:
+    gr.Markdown("# 🔬 Pancreas Cancer Annotations Explorer")
+    gr.Markdown("Hover over highlighted text to see extracted values. `/` means not found.")
 
     with gr.Row():
-        with gr.Column(scale=2):
-            sample_slider = gr.Slider(
-                minimum=0,
-                maximum=len(dataset) - 1,
-                step=1,
-                value=0,
-                label=f"📌 Sample Index (0 - {len(dataset) - 1})",
-                info="Drag to browse samples"
-            )
-        with gr.Column(scale=1):
-            stats_display = gr.Markdown("", elem_classes=["stats-badge"])
+        slider = gr.Slider(0, len(dataset) - 1, value=0, step=1, label="Sample")
 
     with gr.Row():
-        with gr.Column(scale=1):
-            search_box = gr.Textbox(
-                label="🔍 Search",
-                placeholder="Type to search in clinical reports...",
-                info="Min 3 characters"
-            )
-            search_results = gr.Dataframe(
-                headers=["#", "Preview"],
-                label="Results",
-                interactive=False,
-                height=200
-            )
-
-    gr.Markdown("---")
-    gr.Markdown("### 📄 Clinical Report with Entity Highlighting")
-    gr.Markdown("*Hover over colored text to see the extracted variable and value*")
-
-    cr_display = gr.HTML()
-
-    gr.Markdown("---")
-    gr.Markdown("### 📊 Extracted Annotations")
-
-    annotations_display = gr.HTML()
-
-    # Event handlers
-    sample_slider.change(
-        fn=display_sample,
-        inputs=[sample_slider],
-        outputs=[cr_display, annotations_display, stats_display]
-    )
-
-    search_box.change(
-        fn=search_samples,
-        inputs=[search_box],
-        outputs=[search_results]
-    )
-
-    def on_select(evt: gr.SelectData, data):
-        if data is not None and len(data) > 0:
-            try:
-                selected_idx = int(data[evt.index[0]][0])
-                return selected_idx
-            except (ValueError, IndexError, TypeError):
-                pass
-        return 0
-
-    search_results.select(
-        fn=on_select,
-        inputs=[search_results],
-        outputs=[sample_slider]
-    )
-
-    # Load first sample on start
-    demo.load(
-        fn=display_sample,
-        inputs=[sample_slider],
-        outputs=[cr_display, annotations_display, stats_display]
-    )
-
+        with gr.Column():
+            gr.Markdown("### Clinical Report")
+            cr_html = gr.HTML()
+        with gr.Column():
+            gr.Markdown("### Extracted Variables")
+            table_html = gr.HTML()
+
+    slider.change(display_sample, inputs=[slider], outputs=[cr_html, table_html])
+    demo.load(display_sample, inputs=[slider], outputs=[cr_html, table_html])
 
 if __name__ == "__main__":
     demo.launch()