change file and update

app.py

@@ -1,9 +1,8 @@
 import boto3
 import os
 import json
-import re
 import gradio as gr
-from typing import List, Dict, Tuple, Optional, Union, Any
+from typing import List, Dict, Tuple, Optional, Any
 
 # ── S3 CONFIG ─────────────────────────────────────────────────────────────────
 s3 = boto3.client(
@@ -13,18 +12,18 @@ s3 = boto3.client(
     region_name           = os.getenv("AWS_DEFAULT_REGION", "ap-southeast-2"),
 )
 
-# ai4data/datause-annotation
-# S3 bucket and keys
 BUCKET       = "doccano-processed"
-#INIT_KEY     = "gradio/initial_data_train.json"
-INIT_KEY = "gradio/holdout_data_review.json"
-#VALID_PREFIX = "validated_records/"
-VALID_PREFIX = "holdout_data_review_output/"
+INIT_KEY = "gradio/ai4data-revalidate-data.json"
+VALID_PREFIX = "ai4data-revalidate-data-output/"
 
 # ── Helpers to load & save from S3 ──────────────────────────────────────────────
 def load_initial_data() -> List[Dict]:
     obj = s3.get_object(Bucket=BUCKET, Key=INIT_KEY)
-    return json.loads(obj['Body'].read())
+    data = json.loads(obj['Body'].read())
+    # assume ner_text spans use end-index as non-inclusive
+    for rec in data:
+        rec.setdefault("validated", False)
+    return data
 
 def load_all_validations() -> Dict[int, Dict]:
     records = {}
@@ -33,10 +32,10 @@ def load_all_validations() -> Dict[int, Dict]:
     )
     for page in pages:
         for obj in page.get("Contents", []):
-            key = obj["Key"]
-            idx = int(os.path.splitext(os.path.basename(key))[0])
-            data = s3.get_object(Bucket=BUCKET, Key=key)["Body"].read()
-            records[idx] = json.loads(data)
+            idx = int(os.path.splitext(os.path.basename(obj["Key"]))[0])
+            rec = json.loads(s3.get_object(Bucket=BUCKET, Key=obj["Key"])['Body'].read())
+            rec.setdefault("validated", True)
+            records[idx] = rec
     return records
 
 def save_single_validation(idx: int, record: Dict):
@@ -47,14 +46,13 @@ def save_single_validation(idx: int, record: Dict):
         Body        = json.dumps(record, indent=2).encode('utf-8'),
         ContentType = 'application/json'
     )
+    ##fckxk
 
 class DynamicDataset:
     def __init__(self, data: List[Dict]):
         self.data    = data
         self.len     = len(data)
         self.current = 0
-        for ex in self.data:
-            ex.setdefault("validated", False)
 
     def example(self, idx: int) -> Dict:
         self.current = max(0, min(self.len - 1, idx))
@@ -87,161 +85,120 @@ class DynamicDataset:
     def validate(self):
         self.data[self.current]["validated"] = True
 
-def tokenize_text(text: str) -> List[str]:
-    return re.findall(r"\w+(?:[-_]\w+)*|[^\s\w]", text)
-
+# ── Highlight utils using raw text (half-open intervals) ───────────────────────
 def prepare_for_highlight(data: Dict) -> List[Tuple[str, Optional[str]]]:
-    tokens = data["tokenized_text"]
-    ner    = data["ner"]
-    highlighted, curr_ent, ent_buf, norm_buf = [], None, [], []
-    for idx, tok in enumerate(tokens):
-        if curr_ent is None or idx > curr_ent[1]:
-            if ent_buf:
-                highlighted.append((" ".join(ent_buf), curr_ent[2]))
-                ent_buf = []
-            curr_ent = next((e for e in ner if e[0] == idx), None)
-        if curr_ent and curr_ent[0] <= idx <= curr_ent[1]:
-            if norm_buf:
-                highlighted.append((" ".join(norm_buf), None))
-                norm_buf = []
-            ent_buf.append(tok)
-        else:
-            if ent_buf:
-                highlighted.append((" ".join(ent_buf), curr_ent[2]))
-                ent_buf = []
-            norm_buf.append(tok)
-    if ent_buf:
-        highlighted.append((" ".join(ent_buf), curr_ent[2]))
-    if norm_buf:
-        highlighted.append((" ".join(norm_buf), None))
-    return [(re.sub(r"\s(?=[,\.!?…:;])", "", txt), lbl) for txt, lbl in highlighted]
-
-
-def extract_tokens_and_labels(highlighted: List[Dict[str, Union[str, None]]]
-                            ) -> Tuple[List[str], List[Tuple[int,int,str]]]:
-    tokens, ner = [], []
-    token_idx = 0
-
+    text = data.get("text", "")
+    # use annotated spans if any, else original ner_text
+    ner_spans = data.get("ner_annotated", data.get("ner_text", []))
+    segments: List[Tuple[str, Optional[str]]] = []
+    last_idx = 0
+    for start, end, label in sorted(ner_spans, key=lambda x: x[0]):
+        # slice in [start, end) since end is non-inclusive
+        if start > last_idx:
+            segments.append((text[last_idx:start], None))
+        segments.append((text[start:end], label))
+        last_idx = end
+    if last_idx < len(text):
+        segments.append((text[last_idx:], None))
+    return segments
+
+def align_spans_to_text(highlighted: List[Dict[str, Any]], text: str) -> List[Tuple[int, int, str]]:
+    spans: List[Tuple[int, int, str]] = []
+    search_start = 0
     for entry in highlighted:
-        text  = entry['token']
-        label = entry.get('class_or_confidence') or entry.get('class') or entry.get('label')
-        # split into real tokens
-        toks = tokenize_text(text)
-        start = token_idx
-        end   = token_idx + len(toks) - 1
-
-        tokens.extend(toks)
-        if label:
-            ner.append((start, end, label))
-
-        token_idx = end + 1
-
-    return tokens, ner
+        chunk = entry["token"]
+        label = entry.get("class_or_confidence") or entry.get("class") or entry.get("label")
+        pos = text.find(chunk, search_start)
+        if pos >= 0:
+            # new end is start + len(chunk)
+            spans.append((pos, pos + len(chunk), label))
+            search_start = pos + len(chunk)
+        else:
+            print(f"⚠️ Couldn’t align chunk: {chunk!r}")
+    return spans
 
+# ── Gradio demo ────────────────────────────────────────────────────────────────
 def create_demo() -> gr.Blocks:
     data            = load_initial_data()
     validated_store = load_all_validations()
-
-    # mark any pre-validated examples
-    for idx in validated_store:
-        if 0 <= idx < len(data):
-            data[idx]["validated"] = True
-
     dynamic_dataset = DynamicDataset(data)
 
-    def make_info(rec):
+    def make_info(rec: Dict) -> str:
         fn = rec.get("filename", "—")
         pg = rec.get("page", "—")
-        # Markdown with line break for Gradio
-        return f"**File:** `{fn}`  \n**Page:** `{pg}`"
+        sg = rec.get("segment", "—")
+        return f"**File:** `{fn}`  \n**Page:** `{pg}`\n**sSegment:** `{sg}`"
 
-    def align_spans_to_tokens(
-        highlighted: List[Dict[str, Union[str, None]]],
-        tokens: List[str]
-    ) -> List[Tuple[int, int, str]]:
-        """
-        Align each highlighted chunk to the next matching tokens in the list,
-        advancing a pointer so repeated tokens map in the order you clicked them.
-        """
-        spans = []
-        search_start = 0
-
-        for entry in highlighted:
-            text  = entry["token"]
-            label = entry.get("class_or_confidence") or entry.get("label") or entry.get("class")
-            if not label:
-                continue
-
-            chunk_toks = tokenize_text(text)
-            # scan only from the end of the last match
-            for i in range(search_start, len(tokens) - len(chunk_toks) + 1):
-                if tokens[i:i + len(chunk_toks)] == chunk_toks:
-                    spans.append((i, i + len(chunk_toks) - 1, label))
-                    search_start = i + len(chunk_toks)
-                    break
-            else:
-                print(f"⚠️ Couldn’t align chunk: {text!r}")
-
-        return spans
-
-    def load_example(idx):
+    def load_example(idx: int):
+        # If there’s a validated version, show that; otherwise fall back
         rec  = validated_store.get(idx, dynamic_dataset.example(idx))
         segs = prepare_for_highlight(rec)
         return segs, rec.get("validated", False), idx, make_info(rec)
 
     def update_example(highlighted, idx: int):
-        rec = dynamic_dataset.data[idx]
-        # re‐tokenize
-        orig_tokens = tokenize_text(rec["text"])
-        # realign highlights
-        new_ner = align_spans_to_tokens(highlighted, orig_tokens)
-        # overwrite & mark un-validated
-        rec["tokenized_text"] = orig_tokens
-        rec["ner"]            = new_ner
-        rec["validated"]      = False
+        # Always edit the dynamic data, not the validated copy.
+        rec   = dynamic_dataset.data[idx]
+        text  = rec.get("text", "")
+        new_spans = align_spans_to_text(highlighted, text)
+        # store edits as half-open
+        rec["ner_annotated"] = new_spans
+        rec["validated"]    = False
         return prepare_for_highlight(rec), rec["validated"], idx, make_info(rec)
 
     def do_validate(highlighted, idx: int):
-        # in-memory mark
+        # Edit dynamic data first
+        rec   = dynamic_dataset.data[idx]
+        text  = rec.get("text", "")
+        new_spans = align_spans_to_text(highlighted, text)
+        rec["ner_annotated"] = new_spans
         dynamic_dataset.validate()
-        rec = dynamic_dataset.data[idx]
-        orig_tokens = tokenize_text(rec["text"])
-        new_ner = align_spans_to_tokens(highlighted, orig_tokens)
-        rec["tokenized_text"] = orig_tokens
-        rec["ner"]            = new_ner
-        # persist to disk/store
-        save_single_validation(idx, rec)
-        return prepare_for_highlight(rec), True, make_info(rec)
+        # Now push that validated copy to S3 and to validated_store
+        rec_to_save = rec.copy()
+        rec_to_save["validated"] = True
+        save_single_validation(idx, rec_to_save)
+        validated_store[idx] = rec_to_save
+        return prepare_for_highlight(rec_to_save), True, make_info(rec_to_save)
 
     def nav(fn):
-        rec  = fn()
+        # Move the index/cursor in dynamic_dataset
+        _    = fn()
+        idx  = dynamic_dataset.current
+        # If there’s a validated version, show that; else show dynamic data
+        rec  = validated_store.get(idx, dynamic_dataset.data[idx])
         segs = prepare_for_highlight(rec)
-        return segs, rec.get("validated", False), dynamic_dataset.current, make_info(rec)
+        return segs, rec.get("validated", False), idx, make_info(rec)
 
     with gr.Blocks() as demo:
-        prog        = gr.Slider(0, dynamic_dataset.len-1, value=0, step=1, label="Example #", interactive=False)
-        inp_box     = gr.HighlightedText(label="Sentence", interactive=True)
-        info_md     = gr.Markdown(label="Source")      # ← shows filename & page
-        status      = gr.Checkbox(label="Validated?", value=False, interactive=False)
-
-        gr.Markdown(
-            "[📖 Entity Tag Guide](https://huggingface.co/spaces/rafmacalaba/datause-annotation/blob/main/guidelines.md)"
+        prog = gr.Slider(
+            minimum=0,
+            maximum=dynamic_dataset.len - 1,
+            value=0,
+            step=1,
+            label="Example # (slide to navigate)",
+            interactive=True,
         )
+        inp_box = gr.HighlightedText(label="Sentence", interactive=True)
+        info_md = gr.Markdown(label="Source")
+        status  = gr.Checkbox(label="Validated?", value=False, interactive=False)
 
-        with gr.Row():
-            prev_btn    = gr.Button("◀️ Previous")
-            apply_btn   = gr.Button("📝 Apply Changes")
-            next_btn    = gr.Button("Next ▶️")
+        gr.Markdown("[📖 Entity Tag Guide](https://huggingface.co/spaces/rafmacalaba/datause-annotation/blob/main/guidelines.md)")
 
+        with gr.Row():
+            prev_btn  = gr.Button("◀️ Previous")
+            apply_btn = gr.Button("📝 Apply Changes")
+            next_btn  = gr.Button("Next ▶️")
         with gr.Row():
             skip_prev     = gr.Button("⏮️ Prev Unvalidated")
             validate_btn = gr.Button("✅ Validate")
             skip_next     = gr.Button("⏭️ Next Unvalidated")
 
-        # initial load
+        # ───── Wiring events ──────────────────────────────────────────────────
+        prog.release(
+            fn=load_example,
+            inputs=[prog],
+            outputs=[inp_box, status, prog, info_md],
+        )
         demo.load(load_example, inputs=prog, outputs=[inp_box, status, prog, info_md])
-
-        # wire up actions (all now also update info_md)
         apply_btn.click(update_example, inputs=[inp_box, prog], outputs=[inp_box, status, prog, info_md])
         prev_btn.click(lambda: nav(dynamic_dataset.prev), inputs=None, outputs=[inp_box, status, prog, info_md])
         next_btn.click(lambda: nav(dynamic_dataset.next), inputs=None, outputs=[inp_box, status, prog, info_md])
@@ -252,5 +209,4 @@ def create_demo() -> gr.Blocks:
     return demo
 
 if __name__ == "__main__":
-    demo = create_demo()
-    demo.launch(share=True, inline=True, debug=True)
+    create_demo().launch(share=False, debug=True)