v5 phase

app.py

@@ -17,9 +17,9 @@ s3 = boto3.client(
 # S3 bucket and keys
 BUCKET       = "doccano-processed"
 #INIT_KEY     = "gradio/initial_data_train.json"
-INIT_KEY = "gradio/refugee_train_initial_data_v4.json"
+INIT_KEY = "gradio/refugee_train_initial_datav5.json "
 #VALID_PREFIX = "validated_records/"
-VALID_PREFIX = "refugee_train_validated_v4/"
+VALID_PREFIX = "refugee_train_validated_v5/"
 
 # ── Helpers to load & save from S3 ──────────────────────────────────────────────
 def load_initial_data() -> List[Dict]:
@@ -138,142 +138,6 @@ def extract_tokens_and_labels(highlighted: List[Dict[str, Union[str, None]]]
 
     return tokens, ner
 
-
-# ── App factory ────────────────────────────────────────────────────────────────
-# def create_demo() -> gr.Blocks:
-#     data             = load_initial_data()
-#     validated_store  = load_all_validations()
-
-#     for idx in validated_store:
-#         if 0 <= idx < len(data):
-#             data[idx]["validated"] = True
-#     dynamic_dataset  = DynamicDataset(data)
-#     with gr.Blocks() as demo:
-#         prog      = gr.Slider(0, dynamic_dataset.len-1, value=0, step=1, label="Example #", interactive=False)
-#         inp_box   = gr.HighlightedText(label="Sentence", interactive=True)
-#         status    = gr.Checkbox(label="Validated?", value=False, interactive=False)
-#         filename_disp = gr.Markdown(label="Filename")    # NEW: shows current filename
-#         page_disp     = gr.Markdown(label="Page")        # NEW: shows current page number
-#         gr.Markdown(
-#             "[📖 Entity Tag Guide](https://huggingface.co/spaces/rafmacalaba/datause-annotation/blob/main/guidelines.md)"
-#         )
-        
-#         with gr.Row():
-#             prev_btn  = gr.Button("◀️ Previous")
-#             apply_btn = gr.Button("📝 Apply Changes")
-#             next_btn  = gr.Button("Next ▶️")
-#         with gr.Row():
-#             skip_prev = gr.Button("⏮️ Prev Unvalidated")
-#             validate_btn = gr.Button("✅ Validate")
-#             skip_next = gr.Button("⏭️ Next Unvalidated")
-
-#         # def load_example(idx):
-#         #     rec  = validated_store.get(idx, dynamic_dataset.example(idx))
-#         #     segs = prepare_for_highlight(rec)
-#         #     return segs, rec.get("validated", False), idx
-
-#         def load_example(idx):
-#             rec  = validated_store.get(idx, dynamic_dataset.example(idx))
-#             segs = prepare_for_highlight(rec)
-#             return (
-#                 segs,
-#                 rec.get("validated", False),
-#                 idx,
-#                 rec.get("filename", ""),    # <-- returns filename for filename_disp
-#                 f"Page {rec.get('page', '')}"  # <-- returns page for page_disp
-#             )
-
-#         def update_example(highlighted, idx: int):
-#             # grab the record
-#             rec = dynamic_dataset.data[idx]
-
-#             # re‐tokenize from the raw text (same as do_validate)
-#             orig_tokens = tokenize_text(rec["text"])
-
-#             # realign the user's highlights back to those tokens
-#             new_ner = align_spans_to_tokens(highlighted, orig_tokens)
-
-#             # overwrite both token list and span list (and mark un‐validated)
-#             rec["tokenized_text"] = orig_tokens
-#             rec["ner"]            = new_ner
-#             rec["validated"]      = False
-
-#             # re‐render
-#             return prepare_for_highlight(rec)
-
-#         def align_spans_to_tokens(
-#             highlighted: List[Dict[str, Union[str, None]]],
-#             tokens: List[str]
-#         ) -> List[Tuple[int,int,str]]:
-#             """
-#             Align each highlighted chunk to the next matching tokens in the list,
-#             advancing a pointer so repeated tokens map in the order you clicked them.
-#             """
-#             spans = []
-#             search_start = 0
-
-#             for entry in highlighted:
-#                 text  = entry["token"]
-#                 label = entry.get("class_or_confidence") or entry.get("label") or entry.get("class")
-#                 if not label:
-#                     continue
-
-#                 chunk_toks = tokenize_text(text)
-#                 # scan only from the end of the last match
-#                 for i in range(search_start, len(tokens) - len(chunk_toks) + 1):
-#                     if tokens[i:i+len(chunk_toks)] == chunk_toks:
-#                         spans.append((i, i + len(chunk_toks) - 1, label))
-#                         search_start = i + len(chunk_toks)
-#                         break
-#                 else:
-#                     print(f"⚠️ Couldn’t align chunk: {text!r}")
-
-#             return spans
-
-#         def do_validate(highlighted, idx: int):
-#             # mark validated in memory
-#             dynamic_dataset.validate()
-
-#             # grab the record
-#             rec = dynamic_dataset.data[idx]
-
-#             # re-tokenize from the original text
-#             orig_tokens = tokenize_text(rec["text"])
-
-#             # realign the user's highlighted segments to those tokens
-#             new_ner = align_spans_to_tokens(highlighted, orig_tokens)
-
-#             # overwrite both token list and span list
-#             rec["tokenized_text"] = orig_tokens
-#             rec["ner"]            = new_ner
-
-#             # persist
-#             save_single_validation(idx, rec)
-
-#             # re-render and show checkbox checked
-#             return prepare_for_highlight(rec), True
-
-
-#         def nav(fn):
-#             rec  = fn()
-#             segs = prepare_for_highlight(rec)
-#             return segs, rec.get("validated", False), dynamic_dataset.current
-
-#         demo.load(load_example, inputs=prog, outputs=[inp_box, status, prog])
-#         apply_btn.click(
-#             fn=update_example,
-#             inputs=[inp_box, prog],     # pass both the highlights *and* the example idx
-#             outputs=inp_box
-#         )
-#         #apply_btn.click(update_spans, inputs=inp_box, outputs=inp_box)
-#         prev_btn.click(lambda: nav(dynamic_dataset.prev), inputs=None, outputs=[inp_box, status, prog])
-#         validate_btn.click(do_validate, inputs=[inp_box, prog], outputs=[inp_box, status])
-#         next_btn.click(lambda: nav(dynamic_dataset.next), inputs=None, outputs=[inp_box, status, prog])
-#         skip_prev.click(lambda: nav(dynamic_dataset.jump_prev_unvalidated), inputs=None, outputs=[inp_box, status, prog])
-#         skip_next.click(lambda: nav(dynamic_dataset.jump_next_unvalidated), inputs=None, outputs=[inp_box, status, prog])
-
-#     return demo
-
 def create_demo() -> gr.Blocks:
     data            = load_initial_data()
     validated_store = load_all_validations()