File size: 7,032 Bytes
1db7196 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 | import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
import json
import torch
import pickle
import gradio as gr
import textstat
from sentence_transformers import SentenceTransformer, util
# --- Configuration & Paths ---
LANG_CODE = "en"
CHUNKS_PATH = f"/home/mshahidul/readctrl/data/vector_db/db_model/wiki_{LANG_CODE}_chunks.pkl"
EMBS_PATH = f"/home/mshahidul/readctrl/data/vector_db/db_model/wiki_{LANG_CODE}_embs.pt"
TARGET_DOCS_PATH = f"/home/mshahidul/readctrl/data/synthetic_dataset_diff_labels/syn_data_diff_labels_{LANG_CODE}_v1.json"
SAVE_PATH = f"/home/mshahidul/readctrl/data/data_annotator_data/manual_selections_{LANG_CODE}.json"
# --- 1. Load Resources ---
print("Loading Model and Tensors...")
model = SentenceTransformer('all-MiniLM-L6-v2')
with open(CHUNKS_PATH, "rb") as f:
wiki_chunks = pickle.load(f)
device = "cuda" if torch.cuda.is_available() else "cpu"
wiki_embs = torch.load(EMBS_PATH).to(device)
with open(TARGET_DOCS_PATH, "r") as f:
raw_targets = json.load(f)
target_list = []
for item in raw_targets:
for label, text in item['diff_label_texts'].items():
target_list.append({
"index": item['index'],
"label": label,
"text": text
})
# --- 2. Resume Logic ---
def get_resume_index():
"""Finds the first index in target_list that hasn't been saved yet."""
if not os.path.exists(SAVE_PATH):
return 0
try:
with open(SAVE_PATH, "r") as f:
saved_data = json.load(f)
# Create a set of (index, label) tuples that are already done
done_keys = {(d['index'], d['label']) for d in saved_data}
for i, item in enumerate(target_list):
if (item['index'], item['label']) not in done_keys:
return i
return len(target_list) - 1 # All done
except Exception as e:
print(f"Error loading save file: {e}")
return 0
START_INDEX = get_resume_index()
print(f"Resuming from index: {START_INDEX}")
# --- 3. Logic Functions ---
def get_candidates(target_text, top_k=20):
query_emb = model.encode(target_text, convert_to_tensor=True).to(device)
hits = util.semantic_search(query_emb, wiki_embs, top_k=top_k)[0]
candidates = []
for hit in hits:
candidates.append(wiki_chunks[hit['corpus_id']])
return candidates
def calculate_stats(text):
if not text: return "N/A"
wc = len(text.split())
fk = textstat.flesch_kincaid_grade(text)
return f"📏 Words: {wc} | 🎓 FKGL: {fk}"
def save_selection(target_idx, label, original_text, selected_wiki):
entry = {
"index": target_idx,
"label": label,
"original_text": original_text,
"selected_wiki_anchor": selected_wiki,
"wiki_fkgl": textstat.flesch_kincaid_grade(selected_wiki),
"doc_fkgl": textstat.flesch_kincaid_grade(original_text)
}
existing_data = []
if os.path.exists(SAVE_PATH):
try:
with open(SAVE_PATH, "r") as f:
existing_data = json.load(f)
except:
existing_data = []
# Overwrite if exists, otherwise append
existing_data = [d for d in existing_data if not (d['index'] == target_idx and d['label'] == label)]
existing_data.append(entry)
with open(SAVE_PATH, "w") as f:
json.dump(existing_data, f, indent=2)
return f"✅ Saved: ID {target_idx} ({label})"
# --- 4. Gradio UI ---
with gr.Blocks(theme=gr.themes.Soft(), title="Wiki Anchor Selector") as demo:
gr.Markdown(f"# 🔍 ReadCtrl: Anchor Selection (Resume Mode)")
# Initialize state with the calculated START_INDEX
current_idx = gr.State(START_INDEX)
with gr.Row():
with gr.Column(scale=1):
target_info = gr.Markdown("### Loading...")
label_display = gr.Textbox(label="Target Readability Level", interactive=False)
display_text = gr.Textbox(label="Medical Text", lines=12, interactive=False)
target_stats = gr.Markdown("Stats: ...")
with gr.Column(scale=2):
wiki_dropdown = gr.Dropdown(
label="Select Candidate Number",
choices=[],
interactive=True
)
full_wiki_view = gr.Textbox(label="Wikipedia Chunk Preview", lines=12, interactive=False)
wiki_stats = gr.Markdown("Stats: ...")
status_msg = gr.Markdown("### *Status: Ready*")
with gr.Row():
prev_btn = gr.Button("⬅️ Previous")
save_btn = gr.Button("💾 Confirm & Save", variant="primary")
next_btn = gr.Button("Next / Skip ➡️")
def load_item(idx):
if not (0 <= idx < len(target_list)):
return "End", "None", "", "", gr.update(choices=[], value=None), "", "", "Finished all items!"
doc = target_list[idx]
candidates = get_candidates(doc['text'], top_k=20)
info = f"### Document {idx + 1} of {len(target_list)} (ID: {doc['index']})"
t_stats = calculate_stats(doc['text'])
dropdown_choices = [(f"Candidate {i+1}", c) for i, c in enumerate(candidates)]
return (
info,
doc['label'].upper(),
doc['text'],
t_stats,
gr.update(choices=dropdown_choices, value=candidates[0]),
candidates[0],
calculate_stats(candidates[0]),
f"Currently viewing index {idx}"
)
def on_dropdown_change(selected_text):
if not selected_text: return "", ""
return selected_text, calculate_stats(selected_text)
def handle_next(idx):
new_idx = min(len(target_list) - 1, idx + 1)
return [new_idx] + list(load_item(new_idx))
def handle_prev(idx):
new_idx = max(0, idx - 1)
return [new_idx] + list(load_item(new_idx))
# --- Event Bindings ---
# Trigger load_item on page load using the START_INDEX from state
demo.load(load_item, inputs=[current_idx],
outputs=[target_info, label_display, display_text, target_stats, wiki_dropdown, full_wiki_view, wiki_stats, status_msg])
wiki_dropdown.change(on_dropdown_change, inputs=wiki_dropdown, outputs=[full_wiki_view, wiki_stats])
save_btn.click(lambda i, t, w: save_selection(target_list[i]['index'], target_list[i]['label'], t, w),
inputs=[current_idx, display_text, wiki_dropdown],
outputs=[status_msg])
next_btn.click(handle_next, inputs=[current_idx], outputs=[current_idx, target_info, label_display, display_text, target_stats, wiki_dropdown, full_wiki_view, wiki_stats, status_msg])
prev_btn.click(handle_prev, inputs=[current_idx], outputs=[current_idx, target_info, label_display, display_text, target_stats, wiki_dropdown, full_wiki_view, wiki_stats, status_msg])
if __name__ == "__main__":
demo.launch(server_name="0.0.0.0", server_port=7861, share=True) |