Add Review & Correct page: human-in-the-loop entity editing

Click a predicted entity to relabel/delete it, or click any word to tag a
missed entity, then save a corrected version with human-override stats.
Edit state is keyed by a URL id in a process-global store so it survives the
full reload that an in-text <a> click triggers in Streamlit.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app.py

@@ -37,6 +37,9 @@ st.markdown(
 - **🔎 Live Parser** — paste or upload a single CV and watch it get **tokenized and
   classified** in real time: sub-word token chips coloured by predicted label, the
   original text with highlighted entities, and a clean structured summary.
+- **✍️ Review & Correct** — *human in the loop*: click the model's predictions to
+  **relabel or delete** them, click any word to **tag a missed entity**, then save a
+  corrected version with stats on how much the human overrode the model.
 - **📊 Analytics** — upload a batch of CVs (PDF / DOCX / TXT) and the page builds a
   **skills word cloud** plus top Job Titles / Skills / Education charts across the set.
 

lib/review.py

@@ -0,0 +1,106 @@
+"""Human-in-the-loop review logic for the Live Parser output.
+
+Pure functions over the entity list (no Streamlit imports) so the corrected
+highlighted text, the structured summary, the JSON export and the override
+stats all derive from one editable source of truth.
+
+A "work" item mirrors a model entity plus review bookkeeping:
+    {"_id", "type", "start", "end", "text", "conf",
+     "origin": "model"|"added", "orig_type": <type or None>}
+"""
+from __future__ import annotations
+
+import re
+
+
+def seed_review(entities):
+    """Copy model entities into an editable working list with stable ids."""
+    work = []
+    for i, e in enumerate(entities):
+        work.append({
+            "_id": i,
+            "type": e["type"],
+            "start": e["start"],
+            "end": e["end"],
+            "text": e["text"],
+            "conf": e.get("conf", 1.0),
+            "origin": "model",
+            "orig_type": e["type"],   # frozen original label, for the relabel diff
+        })
+    return work
+
+
+def _next_id(work):
+    return max((e["_id"] for e in work), default=-1) + 1
+
+
+def relabel(work, _id, new_type):
+    """Change the label of the entity with id ``_id`` (in place)."""
+    for e in work:
+        if e["_id"] == _id:
+            e["type"] = new_type
+    return work
+
+
+def delete(work, _id):
+    """Remove the entity with id ``_id``."""
+    return [e for e in work if e["_id"] != _id]
+
+
+def add_entity(work, phrase, full_text, etype, hint=0):
+    """Tag ``phrase`` as a new entity, locating it in ``full_text``.
+
+    Returns (work, error). ``error`` is None on success. The phrase is located
+    near ``hint`` (the clicked char offset) first, then anywhere, so repeated
+    words resolve to the one the reviewer clicked.
+    """
+    phrase = (phrase or "").strip()
+    if not phrase:
+        return work, "Nothing to add — the selection was empty."
+    idx = full_text.find(phrase, max(0, hint - len(phrase)))
+    if idx < 0:
+        idx = full_text.find(phrase)
+    if idx < 0:
+        return work, f"Couldn't find “{phrase}” in the CV text."
+    start, end = idx, idx + len(phrase)
+    for e in work:
+        if e["start"] == start and e["end"] == end:
+            return work, "That exact span is already tagged."
+    work = work + [{
+        "_id": _next_id(work),
+        "type": etype, "start": start, "end": end, "text": phrase,
+        "conf": 1.0, "origin": "added", "orig_type": None,
+    }]
+    return work, None
+
+
+def diff_stats(original, corrected):
+    """Compare model output (``original``) with human-corrected output.
+
+    ``original`` is the seeded snapshot; ``corrected`` is the edited work list.
+    """
+    corr_spans = {(e["start"], e["end"]) for e in corrected}
+
+    relabeled, added = [], []
+    for e in corrected:
+        if e.get("origin") == "added":
+            added.append(e)
+        elif e.get("orig_type") and e["type"] != e["orig_type"]:
+            relabeled.append(e)
+
+    deleted = [e for e in original if (e["start"], e["end"]) not in corr_spans]
+
+    n_model = len(original)
+    n_touched = len(relabeled) + len(deleted)   # model entities the human changed
+    return {
+        "n_model": n_model,
+        "n_corrected": len(corrected),
+        "relabeled": relabeled,
+        "deleted": deleted,
+        "added": added,
+        "n_relabeled": len(relabeled),
+        "n_deleted": len(deleted),
+        "n_added": len(added),
+        "n_touched": n_touched,
+        "override_rate": (n_touched / n_model) if n_model else 0.0,
+    }

lib/viz.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import html
+import re
 
 import config
 
@@ -101,6 +102,67 @@ def render_entities_html(text: str, entities: list[dict],
             f'max-height:520px;overflow:auto">{body}</div>')
 
 
+def _clickable_gap(segment: str, base: int, rid: str) -> str:
+    """Render non-entity text where every word links to its 'add' selection.
+
+    ``base`` is the char offset of ``segment`` within the full CV text, so each
+    word's link carries its absolute ``start``/``end`` for the add-entity panel.
+    Links carry ``rid`` because a click is a full reload — the review state is
+    recovered from the process store via this id, not from session_state.
+    """
+    parts = []
+    for m in re.finditer(r"\S+|\s+", segment):
+        chunk = m.group(0)
+        if chunk.isspace():
+            parts.append(html.escape(chunk).replace("\n", "<br>"))
+            continue
+        s, e = base + m.start(), base + m.end()
+        parts.append(
+            f'<a href="?rid={rid}&sel=add-{s}-{e}" target="_self" '
+            f'style="color:inherit;text-decoration:none;border-bottom:1px dotted #c9c9c9;'
+            f'cursor:pointer" title="Tag “{html.escape(chunk)}” as a missed entity">'
+            f'{html.escape(chunk)}</a>'
+        )
+    return "".join(parts)
+
+
+def render_review_html(text: str, entities: list[dict], rid: str) -> str:
+    """Editable variant of the highlighted text for the review page.
+
+    Entities link to ``?rid=…&sel=e-<id>`` (relabel/delete); every other word
+    links to ``?rid=…&sel=add-<start>-<end>`` (tag a missed entity). A click is a
+    full page reload, so the page restores state from the process store by ``rid``.
+    """
+    ents = sorted((e for e in entities if e["type"] in config.ENTITY_COLORS),
+                  key=lambda e: e["start"])
+    out, cursor = [], 0
+    for e in ents:
+        if e["start"] < cursor:  # skip any overlap defensively
+            continue
+        out.append(_clickable_gap(text[cursor:e["start"]], cursor, rid))
+        color = config.ENTITY_COLORS[e["type"]]
+        label = config.ENTITY_LABELS[e["type"]]
+        ring = "box-shadow:0 0 0 2px #fff,0 0 0 3px #555;" if e.get("origin") == "added" else ""
+        out.append(
+            f'<a href="?rid={rid}&sel=e-{e["_id"]}" target="_self" '
+            f'style="background:{color};color:#fff;padding:1px 4px;border-radius:4px;'
+            f'text-decoration:none;cursor:pointer;{ring}" '
+            f'title="Click to relabel or delete">'
+            f'{html.escape(text[e["start"]:e["end"]])}'
+            f'<sub style="font-size:0.6em;opacity:.85"> {label}</sub></a>'
+        )
+        cursor = e["end"]
+    out.append(_clickable_gap(text[cursor:], cursor, rid))
+    body = "".join(out)
+    hint = ('<div style="margin-bottom:8px;font-size:0.8rem;color:#666">'
+            '👆 Click a coloured entity to <b>relabel</b> or <b>delete</b> it, '
+            'or click any plain word to <b>tag a missed entity</b>.</div>')
+    return (_legend() + hint +
+            f'<div style="line-height:2.3;font-family:system-ui;font-size:0.95rem;'
+            f'border:1px solid #ddd;border-radius:8px;padding:16px;'
+            f'max-height:520px;overflow:auto">{body}</div>')
+
+
 def render_tokens_html(tokens: list[dict], limit: int = 400,
                        shade_by_conf: bool = False) -> str:
     """Sub-word token chips, coloured by predicted label — the 'tokenization view'.

pages/1_Live_Parser.py

@@ -51,6 +51,10 @@ if parse:
     c3.metric("Skills", len(grouped["SKILL"]))
     c4.metric("Education", len(grouped["EDUCATION"]))
 
+    st.page_link("pages/4_Review_&_Correct.py",
+                 label="✍️ Review & correct these predictions (human in the loop)",
+                 icon="➡️")
+
     shade = st.toggle(
         "🌡️ Shade by confidence",
         value=False,

pages/4_Review_&_Correct.py

@@ -0,0 +1,233 @@
+"""Review & Correct — human-in-the-loop editing of the model's predictions.
+
+Takes the most recent Live Parser output (st.session_state["parse"]) and lets a
+reviewer click entities to relabel/delete them and click plain words to tag
+missed entities. Saving snapshots the corrected output and shows a side-by-side
+of what the human overrode, with stats.
+
+State note: a click on a highlighted span is a plain ``<a href>``, which is a
+*full page reload* in Streamlit and wipes st.session_state. So the editable
+state lives in a process-global store keyed by a short id (``rid``) carried in
+the URL, which survives the reload. Nothing is written to disk.
+"""
+import json
+import uuid
+
+import streamlit as st
+
+import config
+from lib import review, viz
+from lib.model import group_entities
+from lib.ui import model_selector
+
+st.set_page_config(page_title="Review & Correct", page_icon="✍️", layout="wide")
+
+model_selector()  # keep the shared sidebar picker consistent across pages
+
+st.title("✍️ Review & Correct")
+st.caption("Human in the loop — fix the model's predictions, then save a corrected version.")
+
+ENTITY_TYPES = config.ENTITY_TYPES
+TYPE_LABELS = [config.ENTITY_LABELS[t] for t in ENTITY_TYPES]
+_LABEL2TYPE = {config.ENTITY_LABELS[t]: t for t in ENTITY_TYPES}
+
+
+@st.cache_resource
+def _review_store():
+    """Process-global {rid: state} dict; survives the per-click full reload.
+
+    Distinct rids keep concurrent sessions/users isolated within the process.
+    """
+    return {}
+
+
+def _clear_selection():
+    # Drop only 'sel' — keep 'rid' so state is still recoverable after reload.
+    if "sel" in st.query_params:
+        del st.query_params["sel"]
+
+
+store = _review_store()
+parse = st.session_state.get("parse")
+rid = st.query_params.get("rid")
+state = store.get(rid) if rid else None
+
+# (Re)seed when arriving from the Live Parser with a new/different CV. On a
+# post-click reload there's no session parse, so we keep the stored state.
+if parse is not None and (state is None or state["text"] != parse["text"]):
+    rid = uuid.uuid4().hex[:8]
+    state = {
+        "text": parse["text"],
+        "original": review.seed_review(parse["entities"]),
+        "work": review.seed_review(parse["entities"]),
+        "phase": "edit",
+    }
+    store[rid] = state
+    st.query_params["rid"] = rid
+
+if state is None:
+    st.info("No parsed CV yet. Open **🔎 Live CV Parser**, parse a CV, then come back here.",
+            icon="ℹ️")
+    st.stop()
+
+text = state["text"]
+work = state["work"]
+original = state["original"]
+phase = state["phase"]
+
+
+# ===========================================================================
+# SAVED PHASE — corrected output + human-override stats
+# ===========================================================================
+if phase == "saved":
+    stats = review.diff_stats(original, work)
+
+    if st.button("✏️ Back to editing"):
+        state["phase"] = "edit"
+        st.rerun()
+
+    st.subheader("📊 Human override")
+    m = st.columns(5)
+    m[0].metric("Model entities", stats["n_model"])
+    m[1].metric("Relabeled", stats["n_relabeled"])
+    m[2].metric("Deleted", stats["n_deleted"])
+    m[3].metric("Added", stats["n_added"])
+    m[4].metric("Override rate", f"{stats['override_rate']:.0%}",
+                help="Share of the model's entities the human relabeled or deleted.")
+
+    grouped = group_entities(work)
+    st.subheader("✅ Corrected output")
+    st.markdown(viz.render_entities_html(text, work, shade_by_conf=False),
+                unsafe_allow_html=True)
+
+    cols = st.columns(3)
+    for col, etype in zip(cols, ENTITY_TYPES):
+        with col:
+            st.markdown(f"**{config.ENTITY_LABELS[etype]}**")
+            vals = grouped[etype]
+            if vals:
+                for v in vals:
+                    st.markdown(f"- {v}")
+            else:
+                st.caption("— none —")
+
+    st.download_button(
+        "⬇️ Download corrected JSON",
+        data=json.dumps({config.ENTITY_LABELS[t]: grouped[t] for t in ENTITY_TYPES},
+                        indent=2),
+        file_name="parsed_cv_corrected.json",
+        mime="application/json",
+    )
+
+    with st.expander("🔍 What the human changed", expanded=True):
+        if stats["relabeled"]:
+            st.markdown("**Relabeled**")
+            st.table([
+                {"text": e["text"],
+                 "model said": config.ENTITY_LABELS[e["orig_type"]],
+                 "human says": config.ENTITY_LABELS[e["type"]]}
+                for e in stats["relabeled"]
+            ])
+        if stats["deleted"]:
+            st.markdown("**Deleted (false positives)**")
+            st.table([{"text": e["text"], "model label": config.ENTITY_LABELS[e["type"]]}
+                      for e in stats["deleted"]])
+        if stats["added"]:
+            st.markdown("**Added (missed by the model)**")
+            st.table([{"text": e["text"], "human label": config.ENTITY_LABELS[e["type"]]}
+                      for e in stats["added"]])
+        if not (stats["relabeled"] or stats["deleted"] or stats["added"]):
+            st.caption("No changes — the human accepted the model's output as-is.")
+    st.stop()
+
+
+# ===========================================================================
+# EDIT PHASE — click entities/words to correct them
+# ===========================================================================
+sel = st.query_params.get("sel")
+
+# ---- Selection editor panel (driven by the ?sel= query param) --------------
+if sel and sel.startswith("e-"):
+    try:
+        sid = int(sel[2:])
+    except ValueError:
+        sid = None
+    ent = next((e for e in work if e["_id"] == sid), None)
+    if ent is None:
+        st.info("That entity was removed. Click another to edit.")
+    else:
+        with st.container(border=True):
+            st.markdown(f"**Editing:** “{ent['text']}” "
+                        f"· model said *{config.ENTITY_LABELS[ent['orig_type']]}*"
+                        if ent.get("orig_type")
+                        else f"**Editing:** “{ent['text']}” · *(human-added)*")
+            c1, c2, c3 = st.columns([2, 1, 1])
+            with c1:
+                new_label = st.selectbox(
+                    "Label", TYPE_LABELS,
+                    index=ENTITY_TYPES.index(ent["type"]), key=f"relabel_{sid}")
+            with c2:
+                st.write("")
+                if st.button("✅ Apply", use_container_width=True, key=f"apply_{sid}"):
+                    review.relabel(state["work"], sid, _LABEL2TYPE[new_label])
+                    _clear_selection()
+                    st.rerun()
+            with c3:
+                st.write("")
+                if st.button("🗑️ Delete", use_container_width=True, key=f"del_{sid}"):
+                    state["work"] = review.delete(state["work"], sid)
+                    _clear_selection()
+                    st.rerun()
+
+elif sel and sel.startswith("add-"):
+    try:
+        _, s, e = sel.split("-")
+        s, e = int(s), int(e)
+    except ValueError:
+        s = e = None
+    if s is None:
+        st.info("Couldn't read that selection. Click a word to try again.")
+    else:
+        with st.container(border=True):
+            st.markdown("**Tag a missed entity** — edit the phrase to extend it across words.")
+            c1, c2, c3 = st.columns([2, 1, 1])
+            with c1:
+                phrase = st.text_input("Phrase", value=text[s:e], key=f"add_phrase_{s}")
+            with c2:
+                add_label = st.selectbox("Label", TYPE_LABELS, key=f"add_label_{s}")
+            with c3:
+                st.write("")
+                if st.button("➕ Add", use_container_width=True, key=f"add_{s}"):
+                    new_work, err = review.add_entity(
+                        state["work"], phrase, text, _LABEL2TYPE[add_label], hint=s)
+                    if err:
+                        st.error(err)
+                    else:
+                        state["work"] = new_work
+                        _clear_selection()
+                        st.rerun()
+
+# ---- Live counts -----------------------------------------------------------
+grouped = group_entities(work)
+c1, c2, c3, c4 = st.columns(4)
+c1.metric("Entities", len(work))
+c2.metric("Job Titles", len(grouped["JOB_TITLE"]))
+c3.metric("Skills", len(grouped["SKILL"]))
+c4.metric("Education", len(grouped["EDUCATION"]))
+
+# ---- The clickable CV ------------------------------------------------------
+st.markdown(viz.render_review_html(text, work, rid), unsafe_allow_html=True)
+
+# ---- Actions ---------------------------------------------------------------
+a1, a2, _ = st.columns([1, 1, 3])
+with a1:
+    if st.button("✅ Save corrections", type="primary", use_container_width=True):
+        state["phase"] = "saved"
+        _clear_selection()
+        st.rerun()
+with a2:
+    if st.button("↩️ Reset to model output", use_container_width=True):
+        # Restore from the frozen model snapshot (no session parse after reloads).
+        state["work"] = [dict(e) for e in original]
+        _clear_selection()
+        st.rerun()