Hugging Face's logo

shahidul034
/
readctrl

Safetensors
Model card Files
xet
Community
readctrl
File size: 4,500 Bytes
9c6961c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
 
import gradio as gr
import json
import os

# -----------------------------
# CONFIGURATION & DATA LOADING
# -----------------------------
EVAL_FILE = "/home/mshahidul/readctrl/data/factual_testing/full_details_evaluation_0_20_qwen3-32B_v2.json"
SOURCE_FILE = "/home/mshahidul/readctrl/data/extracting_subclaim/extracted_subclaims_syn_data_with_gs_summary_en.json"
SAVE_PATH = "/home/mshahidul/readctrl/data/human_eval_results.json"

def load_data():
    with open(EVAL_FILE, 'r') as f:
        eval_data = json.load(f)
    with open(SOURCE_FILE, 'r') as f:
        source_data = json.load(f)
    
    # Create a mapping for quick lookup
    source_map = {item['index']: item for item in source_data}
    return eval_data, source_map

eval_results, source_lookup = load_data()
human_feedback = []

# -----------------------------
# LOGIC
# -----------------------------

def get_record(index):
    entry = eval_results[index]
    idx_val = entry['index']
    source_item = source_lookup.get(idx_val, {})
    
    # Literacy Levels available in this entry
    levels = list(entry['literacy_levels'].keys())
    return entry, source_item, levels
def update_ui(record_idx, level):
    entry, source_item, _ = get_record(record_idx)
    level_data = entry['literacy_levels'][level]
    
    gen_text = source_item.get('diff_label_texts', {}).get(level, "Text not found.")
    
    # Extract missing subclaims as lists of strings
    missing_from_ref = [
        d['source_fact'] for d in level_data['details']['completeness'] 
        if d['status'] == 'not_supported'
    ]
    
    missing_from_full = [
        d['source_subclaim'] for d in level_data['details'].get('source_coverage', []) 
        if d['status'] == 'not_supported'
    ]

    # Return the lists directly to the CheckboxGroup components
    return (
        gen_text, 
        gr.update(choices=missing_from_ref, value=[]), 
        gr.update(choices=missing_from_full, value=[])
    )

def save_judgment(record_idx, level, selected_ref, selected_full, comments):
    entry = eval_results[record_idx]
    result = {
        "index": entry['index'],
        "label": level,
        "unacceptable_ref_claims": selected_ref,   # These are the claims the user "ticked"
        "unacceptable_full_claims": selected_full,
        "comments": comments
    }
    human_feedback.append(result)
    with open(SAVE_PATH, 'w') as f:
        json.dump(human_feedback, f, indent=2)
    return f"Saved judgment for index {entry['index']} ({level})"

# -----------------------------
# UPDATED GRADIO INTERFACE
# -----------------------------

with gr.Blocks(theme=gr.themes.Soft()) as demo:
    gr.Markdown("# 🩺 Medical Summary: Human Evaluation of Information Loss")
    gr.Markdown("Select the specific subclaims that constitute an **unacceptable omission** for this literacy level.")
    
    with gr.Row():
        record_num = gr.Number(label="Record Index (0 to 19)", value=0, precision=0)
        lit_level = gr.Dropdown(
            choices=['low_health_literacy', 'intermediate_health_literacy', 'proficient_health_literacy'], 
            label="Target Literacy Level", 
            value='low_health_literacy'
        )
    
    gr.Markdown("### Generated Text")
    display_text = gr.Textbox(interactive=False, show_label=False, lines=5)
    
    with gr.Row():
        with gr.Column():
            gr.Markdown("### Missing from Reference Summary")
            # Changed from HTML to CheckboxGroup
            ref_check = gr.CheckboxGroup(label="Select Unacceptable Omissions", choices=[])
            
        with gr.Column():
            gr.Markdown("### Missing from Full Source Text")
            # Changed from HTML to CheckboxGroup
            full_check = gr.CheckboxGroup(label="Select Unacceptable Omissions", choices=[])

    comment_box = gr.Textbox(label="Additional Notes (Optional)")
    submit_btn = gr.Button("Save Judgment", variant="primary")
    status_msg = gr.Markdown("")

    # Event Listeners
    record_num.change(update_ui, inputs=[record_num, lit_level], outputs=[display_text, ref_check, full_check])
    lit_level.change(update_ui, inputs=[record_num, lit_level], outputs=[display_text, ref_check, full_check])
    
    submit_btn.click(
        save_judgment, 
        inputs=[record_num, lit_level, ref_check, full_check, comment_box], 
        outputs=status_msg
    )
    
    demo.load(update_ui, inputs=[record_num, lit_level], outputs=[display_text, ref_check, full_check])

demo.launch(share=True)