Spaces:
Sleeping
Sleeping
Create app.py
Browse files
app.py
ADDED
|
@@ -0,0 +1,645 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
import os
|
| 3 |
+
import json
|
| 4 |
+
import random
|
| 5 |
+
import datetime as dt
|
| 6 |
+
from typing import Dict, List, Tuple, Optional
|
| 7 |
+
|
| 8 |
+
import gradio as gr
|
| 9 |
+
|
| 10 |
+
# ------------------------------
|
| 11 |
+
# Paths (override with env vars)
|
| 12 |
+
# ------------------------------
|
| 13 |
+
QA_PATH = os.getenv("QA_PATH", "./spatial_qa_output.json")
|
| 14 |
+
VALIDATION_PATH = os.getenv("VALIDATION_PATH", "./validation_reports_output.json")
|
| 15 |
+
ASSIGNMENTS_PATH = os.getenv("ASSIGNMENTS_PATH", "/data/assignments.json")
|
| 16 |
+
PROGRESS_PATH = os.getenv("PROGRESS_PATH", "/data/progress.json")
|
| 17 |
+
USERS_PATH = os.getenv("USERS_PATH", "./users.json")
|
| 18 |
+
EXPORT_DIR = os.getenv("EXPORT_DIR", "/data")
|
| 19 |
+
|
| 20 |
+
# ------------------------------
|
| 21 |
+
# Utilities
|
| 22 |
+
# ------------------------------
|
| 23 |
+
def _safe_read_json(path: str, default):
|
| 24 |
+
try:
|
| 25 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 26 |
+
return json.load(f)
|
| 27 |
+
except Exception:
|
| 28 |
+
return default
|
| 29 |
+
|
| 30 |
+
def _safe_write_json(path: str, obj):
|
| 31 |
+
"""Safely write JSON file, with fallback to in-memory storage if writing fails."""
|
| 32 |
+
try:
|
| 33 |
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
| 34 |
+
tmp = path + ".tmp"
|
| 35 |
+
with open(tmp, "w", encoding="utf-8") as f:
|
| 36 |
+
json.dump(obj, f, indent=2, ensure_ascii=False)
|
| 37 |
+
os.replace(tmp, path)
|
| 38 |
+
return True
|
| 39 |
+
except (PermissionError, OSError, IOError) as e:
|
| 40 |
+
print(f"Warning: Could not write to {path}: {e}")
|
| 41 |
+
print("Running in read-only mode - data will not persist between sessions")
|
| 42 |
+
return False
|
| 43 |
+
|
| 44 |
+
# ------------------------------
|
| 45 |
+
# In-memory storage for read-only environments
|
| 46 |
+
# ------------------------------
|
| 47 |
+
_in_memory_assignments = None
|
| 48 |
+
_in_memory_progress = None
|
| 49 |
+
_file_write_enabled = True
|
| 50 |
+
|
| 51 |
+
def _get_assignments():
|
| 52 |
+
"""Get assignments from file or in-memory storage."""
|
| 53 |
+
global _in_memory_assignments
|
| 54 |
+
if _in_memory_assignments is not None:
|
| 55 |
+
return _in_memory_assignments
|
| 56 |
+
return _safe_read_json(ASSIGNMENTS_PATH, {})
|
| 57 |
+
|
| 58 |
+
def _set_assignments(assignments):
|
| 59 |
+
"""Set assignments to file and/or in-memory storage."""
|
| 60 |
+
global _in_memory_assignments, _file_write_enabled
|
| 61 |
+
_in_memory_assignments = assignments
|
| 62 |
+
if _file_write_enabled:
|
| 63 |
+
success = _safe_write_json(ASSIGNMENTS_PATH, assignments)
|
| 64 |
+
if not success:
|
| 65 |
+
_file_write_enabled = False
|
| 66 |
+
|
| 67 |
+
def _get_progress():
|
| 68 |
+
"""Get progress from file or in-memory storage."""
|
| 69 |
+
global _in_memory_progress
|
| 70 |
+
if _in_memory_progress is not None:
|
| 71 |
+
return _in_memory_progress
|
| 72 |
+
return _safe_read_json(PROGRESS_PATH, {})
|
| 73 |
+
|
| 74 |
+
def _set_progress(progress):
|
| 75 |
+
"""Set progress to file and/or in-memory storage."""
|
| 76 |
+
global _in_memory_progress, _file_write_enabled
|
| 77 |
+
_in_memory_progress = progress
|
| 78 |
+
if _file_write_enabled:
|
| 79 |
+
success = _safe_write_json(PROGRESS_PATH, progress)
|
| 80 |
+
if not success:
|
| 81 |
+
_file_write_enabled = False
|
| 82 |
+
|
| 83 |
+
# ------------------------------
|
| 84 |
+
# Load data
|
| 85 |
+
# ------------------------------
|
| 86 |
+
def load_data() -> Dict[str, Dict]:
|
| 87 |
+
"""Return dict keyed by instance_id with:
|
| 88 |
+
- findings (str)
|
| 89 |
+
- impressions (str)
|
| 90 |
+
- qa_pairs (list of {'question','answer'})"""
|
| 91 |
+
with open(QA_PATH, "r", encoding="utf-8") as f:
|
| 92 |
+
qa_data = json.load(f)
|
| 93 |
+
with open(VALIDATION_PATH, "r", encoding="utf-8") as f:
|
| 94 |
+
val_data = json.load(f)
|
| 95 |
+
|
| 96 |
+
data = {}
|
| 97 |
+
missing_in_val = []
|
| 98 |
+
for inst_id, payload in qa_data.items():
|
| 99 |
+
if inst_id not in val_data:
|
| 100 |
+
missing_in_val.append(inst_id)
|
| 101 |
+
continue
|
| 102 |
+
find_str = (
|
| 103 |
+
val_data[inst_id].get("Findings_EN")
|
| 104 |
+
or val_data[inst_id].get("Findings")
|
| 105 |
+
or ""
|
| 106 |
+
)
|
| 107 |
+
impr_str = (
|
| 108 |
+
val_data[inst_id].get("Impressions_EN")
|
| 109 |
+
or val_data[inst_id].get("Impressions")
|
| 110 |
+
or ""
|
| 111 |
+
)
|
| 112 |
+
pairs = payload.get("qa_pairs", [])
|
| 113 |
+
# normalize
|
| 114 |
+
normalized_pairs = []
|
| 115 |
+
for p in pairs:
|
| 116 |
+
normalized_pairs.append(
|
| 117 |
+
{
|
| 118 |
+
"question": str(p.get("question", "")).strip(),
|
| 119 |
+
"answer": str(p.get("answer", "")).strip(),
|
| 120 |
+
}
|
| 121 |
+
)
|
| 122 |
+
data[inst_id] = {
|
| 123 |
+
"findings": find_str.strip(),
|
| 124 |
+
"impressions": impr_str.strip(),
|
| 125 |
+
"qa_pairs": normalized_pairs,
|
| 126 |
+
}
|
| 127 |
+
if not data:
|
| 128 |
+
raise RuntimeError("No overlapping instances between QA and Validation files. "
|
| 129 |
+
"Check the JSON files and their keys.")
|
| 130 |
+
return data
|
| 131 |
+
|
| 132 |
+
DATA = load_data()
|
| 133 |
+
INSTANCE_IDS = sorted(list(DATA.keys()))
|
| 134 |
+
|
| 135 |
+
def load_users() -> List[str]:
|
| 136 |
+
"""Load users from JSON file, fallback to default if file doesn't exist."""
|
| 137 |
+
users_data = _safe_read_json(USERS_PATH, {})
|
| 138 |
+
if "users" in users_data and isinstance(users_data["users"], list):
|
| 139 |
+
return [str(user).strip() for user in users_data["users"] if user and str(user).strip()]
|
| 140 |
+
# Fallback to default users
|
| 141 |
+
return [f"user_{i+1:02d}" for i in range(20)]
|
| 142 |
+
|
| 143 |
+
def get_default_seed() -> int:
|
| 144 |
+
"""Load default seed from users JSON file."""
|
| 145 |
+
users_data = _safe_read_json(USERS_PATH, {})
|
| 146 |
+
return users_data.get("default_seed", 42)
|
| 147 |
+
|
| 148 |
+
DEFAULT_USERS = load_users()
|
| 149 |
+
DEFAULT_SEED = get_default_seed()
|
| 150 |
+
|
| 151 |
+
# ----------------------------------
|
| 152 |
+
# Assignment & Progress persistence
|
| 153 |
+
# ----------------------------------
|
| 154 |
+
def init_or_load_assignments(default_users: List[str], seed: int = 42) -> Dict[str, List[str]]:
|
| 155 |
+
"""Load existing assignments or create a balanced random split."""
|
| 156 |
+
assignments = _get_assignments()
|
| 157 |
+
if assignments:
|
| 158 |
+
# filter out instances no longer present; preserve order
|
| 159 |
+
for u, lst in list(assignments.items()):
|
| 160 |
+
assignments[u] = [x for x in lst if x in INSTANCE_IDS]
|
| 161 |
+
_set_assignments(assignments)
|
| 162 |
+
return assignments
|
| 163 |
+
|
| 164 |
+
return create_assignments(default_users, seed)
|
| 165 |
+
|
| 166 |
+
def create_assignments(usernames: List[str], seed: int) -> Dict[str, List[str]]:
|
| 167 |
+
usernames = [u.strip() for u in usernames if u and u.strip()]
|
| 168 |
+
if len(usernames) == 0:
|
| 169 |
+
raise gr.Error("Please provide at least one username.")
|
| 170 |
+
|
| 171 |
+
# Each user gets exactly 10 instances
|
| 172 |
+
instances_per_user = 10
|
| 173 |
+
total_instances_needed = len(usernames) * instances_per_user
|
| 174 |
+
|
| 175 |
+
if total_instances_needed > len(INSTANCE_IDS):
|
| 176 |
+
raise gr.Error(f"Not enough instances available. Need {total_instances_needed} but only have {len(INSTANCE_IDS)}.")
|
| 177 |
+
|
| 178 |
+
insts = INSTANCE_IDS.copy()
|
| 179 |
+
rng = random.Random(int(seed))
|
| 180 |
+
rng.shuffle(insts)
|
| 181 |
+
|
| 182 |
+
# Take only the number of instances we need
|
| 183 |
+
selected_insts = insts[:total_instances_needed]
|
| 184 |
+
|
| 185 |
+
buckets = [[] for _ in usernames]
|
| 186 |
+
for i, inst in enumerate(selected_insts):
|
| 187 |
+
buckets[i % len(usernames)].append(inst)
|
| 188 |
+
|
| 189 |
+
assignments = {usernames[i]: buckets[i] for i in range(len(usernames))}
|
| 190 |
+
_set_assignments(assignments)
|
| 191 |
+
return assignments
|
| 192 |
+
|
| 193 |
+
ASSIGNMENTS = init_or_load_assignments(DEFAULT_USERS, seed=DEFAULT_SEED)
|
| 194 |
+
|
| 195 |
+
def available_users() -> List[str]:
|
| 196 |
+
return sorted(list(_get_assignments().keys()))
|
| 197 |
+
|
| 198 |
+
def load_progress() -> Dict:
|
| 199 |
+
return _get_progress()
|
| 200 |
+
|
| 201 |
+
PROGRESS = load_progress()
|
| 202 |
+
|
| 203 |
+
def _ensure_user_progress_struct(user: str):
|
| 204 |
+
"""Initialize user progress skeleton for new users/instances."""
|
| 205 |
+
global PROGRESS
|
| 206 |
+
if user not in PROGRESS:
|
| 207 |
+
PROGRESS[user] = {}
|
| 208 |
+
# ensure entries exist for assigned instances
|
| 209 |
+
for inst in ASSIGNMENTS.get(user, []):
|
| 210 |
+
n = len(DATA[inst]["qa_pairs"])
|
| 211 |
+
if inst not in PROGRESS[user]:
|
| 212 |
+
PROGRESS[user][inst] = {
|
| 213 |
+
"answers": [None] * n
|
| 214 |
+
}
|
| 215 |
+
else:
|
| 216 |
+
# pad or trim if needed
|
| 217 |
+
ans = PROGRESS[user][inst].get("answers", [])
|
| 218 |
+
if len(ans) < n:
|
| 219 |
+
ans = ans + [None] * (n - len(ans))
|
| 220 |
+
elif len(ans) > n:
|
| 221 |
+
ans = ans[:n]
|
| 222 |
+
PROGRESS[user][inst]["answers"] = ans
|
| 223 |
+
_set_progress(PROGRESS)
|
| 224 |
+
|
| 225 |
+
def save_eval(user: str, inst: str, q_idx: int,
|
| 226 |
+
relevant_choice: Optional[str], correct_choice: Optional[str], note: str = "") -> str:
|
| 227 |
+
"""Persist evaluation for a single QA pair."""
|
| 228 |
+
if not user or not inst:
|
| 229 |
+
return "Select a user to begin."
|
| 230 |
+
_ensure_user_progress_struct(user)
|
| 231 |
+
global PROGRESS
|
| 232 |
+
if inst not in PROGRESS[user]:
|
| 233 |
+
PROGRESS[user][inst] = {"answers": [None] * len(DATA[inst]["qa_pairs"])}
|
| 234 |
+
# map choices to booleans
|
| 235 |
+
rel = None
|
| 236 |
+
if relevant_choice == "β Relevant":
|
| 237 |
+
rel = True
|
| 238 |
+
elif relevant_choice == "β Not relevant":
|
| 239 |
+
rel = False
|
| 240 |
+
|
| 241 |
+
corr = None
|
| 242 |
+
if rel is True:
|
| 243 |
+
if correct_choice == "β Correct":
|
| 244 |
+
corr = True
|
| 245 |
+
elif correct_choice == "β Incorrect":
|
| 246 |
+
corr = False
|
| 247 |
+
# build record
|
| 248 |
+
record = {
|
| 249 |
+
"relevant": rel,
|
| 250 |
+
"correct": corr if rel is True else None,
|
| 251 |
+
"note": (note or "").strip(),
|
| 252 |
+
"saved_at": dt.datetime.utcnow().isoformat() + "Z"
|
| 253 |
+
}
|
| 254 |
+
# save
|
| 255 |
+
answers = PROGRESS[user][inst]["answers"]
|
| 256 |
+
while q_idx >= len(answers): # safety
|
| 257 |
+
answers.append(None)
|
| 258 |
+
answers[q_idx] = record
|
| 259 |
+
_set_progress(PROGRESS)
|
| 260 |
+
|
| 261 |
+
label = f"Saved: {user} β’ {inst} β’ Q{q_idx+1} β relevant={rel}"
|
| 262 |
+
if rel is True:
|
| 263 |
+
label += f", correct={corr}"
|
| 264 |
+
return label
|
| 265 |
+
|
| 266 |
+
def summarize_user(user: str) -> Tuple[str, List[List]]:
|
| 267 |
+
"""Return summary text & table for a user's progress."""
|
| 268 |
+
if not user:
|
| 269 |
+
return ("β", [])
|
| 270 |
+
_ensure_user_progress_struct(user)
|
| 271 |
+
assignments = _get_assignments()
|
| 272 |
+
progress = _get_progress()
|
| 273 |
+
total = 0
|
| 274 |
+
done = 0
|
| 275 |
+
rows = []
|
| 276 |
+
for inst in assignments.get(user, []):
|
| 277 |
+
qa_n = len(DATA[inst]["qa_pairs"])
|
| 278 |
+
total += qa_n
|
| 279 |
+
answers = progress[user][inst]["answers"]
|
| 280 |
+
c_done = sum(1 for a in answers if a is not None)
|
| 281 |
+
done += c_done
|
| 282 |
+
rows.append([inst, c_done, qa_n])
|
| 283 |
+
txt = f"Progress: {done} / {total} QA pairs completed across {len(assignments.get(user, []))} assigned instances."
|
| 284 |
+
return txt, rows
|
| 285 |
+
|
| 286 |
+
def next_unfinished(user: str) -> Tuple[Optional[str], Optional[int]]:
|
| 287 |
+
"""Return (instance_id, q_index) for the next unfinished QA pair for the user."""
|
| 288 |
+
_ensure_user_progress_struct(user)
|
| 289 |
+
assignments = _get_assignments()
|
| 290 |
+
progress = _get_progress()
|
| 291 |
+
for inst in assignments.get(user, []):
|
| 292 |
+
answers = progress[user][inst]["answers"]
|
| 293 |
+
for i, a in enumerate(answers):
|
| 294 |
+
if a is None:
|
| 295 |
+
return inst, i
|
| 296 |
+
return None, None
|
| 297 |
+
|
| 298 |
+
def first_unfinished_in_instance(user: str, inst: str) -> int:
|
| 299 |
+
_ensure_user_progress_struct(user)
|
| 300 |
+
progress = _get_progress()
|
| 301 |
+
answers = progress[user][inst]["answers"]
|
| 302 |
+
for i, a in enumerate(answers):
|
| 303 |
+
if a is None:
|
| 304 |
+
return i
|
| 305 |
+
return 0
|
| 306 |
+
|
| 307 |
+
def get_payload(inst: str, q_idx: int) -> Tuple[str, str, str, str, str]:
|
| 308 |
+
"""Return Q, A, findings, impressions, header text."""
|
| 309 |
+
pairs = DATA[inst]["qa_pairs"]
|
| 310 |
+
n = len(pairs)
|
| 311 |
+
if n == 0:
|
| 312 |
+
q = ""
|
| 313 |
+
a = ""
|
| 314 |
+
header = f"{inst} β No questions (0/0)"
|
| 315 |
+
f = DATA[inst]["findings"]
|
| 316 |
+
im = DATA[inst]["impressions"]
|
| 317 |
+
return q, a, f, im, header
|
| 318 |
+
q_idx = max(0, min(q_idx, n-1))
|
| 319 |
+
q = pairs[q_idx]["question"]
|
| 320 |
+
a = pairs[q_idx]["answer"]
|
| 321 |
+
f = DATA[inst]["findings"]
|
| 322 |
+
im = DATA[inst]["impressions"]
|
| 323 |
+
header = f"{inst} β Question {q_idx+1} / {n}"
|
| 324 |
+
return q, a, f, im, header
|
| 325 |
+
|
| 326 |
+
def export_user_results(user: str) -> str:
|
| 327 |
+
"""Write a CSV + JSON export for the selected user and return a status string & file paths."""
|
| 328 |
+
if not user:
|
| 329 |
+
return "Select a user to export results."
|
| 330 |
+
_ensure_user_progress_struct(user)
|
| 331 |
+
assignments = _get_assignments()
|
| 332 |
+
progress = _get_progress()
|
| 333 |
+
|
| 334 |
+
# Build flat rows
|
| 335 |
+
rows = []
|
| 336 |
+
for inst in assignments.get(user, []):
|
| 337 |
+
pairs = DATA[inst]["qa_pairs"]
|
| 338 |
+
answers = progress[user][inst]["answers"]
|
| 339 |
+
for i in range(len(pairs)):
|
| 340 |
+
ans = answers[i]
|
| 341 |
+
rows.append({
|
| 342 |
+
"user": user,
|
| 343 |
+
"instance_id": inst,
|
| 344 |
+
"q_index": i+1,
|
| 345 |
+
"question": pairs[i]["question"],
|
| 346 |
+
"answer": pairs[i]["answer"],
|
| 347 |
+
"relevant": None if ans is None else ans.get("relevant"),
|
| 348 |
+
"correct": None if ans is None else ans.get("correct"),
|
| 349 |
+
"note": None if ans is None else ans.get("note"),
|
| 350 |
+
"saved_at": None if ans is None else ans.get("saved_at"),
|
| 351 |
+
})
|
| 352 |
+
|
| 353 |
+
# Try to export to files, fallback to in-memory if not possible
|
| 354 |
+
try:
|
| 355 |
+
ts = dt.datetime.utcnow().strftime("%Y%m%d-%H%M%S")
|
| 356 |
+
json_path = os.path.join(EXPORT_DIR, f"results_{user}_{ts}.json")
|
| 357 |
+
csv_path = os.path.join(EXPORT_DIR, f"results_{user}_{ts}.csv")
|
| 358 |
+
|
| 359 |
+
_safe_write_json(json_path, rows)
|
| 360 |
+
# Write CSV
|
| 361 |
+
import csv
|
| 362 |
+
with open(csv_path, "w", newline="", encoding="utf-8") as f:
|
| 363 |
+
writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
|
| 364 |
+
writer.writeheader()
|
| 365 |
+
writer.writerows(rows)
|
| 366 |
+
return f"Exported {len(rows)} rows.\nJSON: {json_path}\nCSV: {csv_path}"
|
| 367 |
+
except (PermissionError, OSError, IOError):
|
| 368 |
+
# Fallback: return data as JSON string
|
| 369 |
+
import json
|
| 370 |
+
json_str = json.dumps(rows, indent=2, ensure_ascii=False)
|
| 371 |
+
return f"Export data (read-only mode):\n\n{json_str[:1000]}{'...' if len(json_str) > 1000 else ''}"
|
| 372 |
+
|
| 373 |
+
# ------------------------------
|
| 374 |
+
# Gradio UI
|
| 375 |
+
# ------------------------------
|
| 376 |
+
|
| 377 |
+
with gr.Blocks(title="Spatial QA Validator", theme=gr.themes.Glass()) as demo:
|
| 378 |
+
# Check if running in read-only mode
|
| 379 |
+
read_only_status = ""
|
| 380 |
+
if not _file_write_enabled:
|
| 381 |
+
read_only_status = "\n\nβ οΈ **Running in read-only mode** - Progress will not persist between sessions"
|
| 382 |
+
|
| 383 |
+
gr.Markdown("## Spatial QA Validation Tool\n"
|
| 384 |
+
"Left: Findings & Impression (Ground Truth)\n\n"
|
| 385 |
+
"Right: Spatial QA pairs (Q/A) to validate.\n\n"
|
| 386 |
+
"For each Q/A:\n"
|
| 387 |
+
"1) Mark if the **Question is relevant** to the Findings/Impression.\n"
|
| 388 |
+
"2) If **relevant**, mark whether the **Answer is correct**.\n" + read_only_status)
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
with gr.Row():
|
| 392 |
+
with gr.Column(scale=1, min_width=260):
|
| 393 |
+
user_dd = gr.Dropdown(choices=available_users(), label="Select user", interactive=True)
|
| 394 |
+
load_btn = gr.Button("Load my queue", variant="primary")
|
| 395 |
+
progress_text = gr.Markdown("")
|
| 396 |
+
progress_table = gr.Dataframe(headers=["Instance", "Done", "Total"], row_count=0, interactive=False)
|
| 397 |
+
|
| 398 |
+
inst_dd = gr.Dropdown(choices=[], label="Assigned instance", interactive=True, visible=False)
|
| 399 |
+
q_slider = gr.Slider(1, 10, value=1, step=1, label="Question #", interactive=True, visible=False)
|
| 400 |
+
|
| 401 |
+
export_btn = gr.Button("Export my results")
|
| 402 |
+
|
| 403 |
+
with gr.Column(scale=2, min_width=600):
|
| 404 |
+
# Left panel: Findings/Impressions
|
| 405 |
+
with gr.Row():
|
| 406 |
+
with gr.Column(scale=1, elem_classes=["left-panel"]):
|
| 407 |
+
findings_tb = gr.Textbox(label="Findings (Ground Truth)", lines=16, interactive=False)
|
| 408 |
+
impressions_tb = gr.Textbox(label="Impression (Ground Truth)", lines=6, interactive=False)
|
| 409 |
+
with gr.Column(scale=1, elem_classes=["left-panel"]):
|
| 410 |
+
header_md = gr.Markdown("")
|
| 411 |
+
question_md = gr.Markdown("")
|
| 412 |
+
answer_md = gr.Markdown("")
|
| 413 |
+
relevant_radio = gr.Radio(choices=["β Relevant", "β Not relevant"],
|
| 414 |
+
label="1) Is the QUESTION relevant to Findings/Impression?",
|
| 415 |
+
interactive=True)
|
| 416 |
+
correct_radio = gr.Radio(choices=["β Correct", "β Incorrect"],
|
| 417 |
+
label="2) If relevant, is the ANSWER correct?",
|
| 418 |
+
interactive=False)
|
| 419 |
+
note_tb = gr.Textbox(label="Optional note", lines=2, placeholder="Any comments...")
|
| 420 |
+
with gr.Row():
|
| 421 |
+
save_btn = gr.Button("Save", variant="secondary")
|
| 422 |
+
save_next_btn = gr.Button("Save & Next", variant="primary")
|
| 423 |
+
skip_btn = gr.Button("Skip to next unfinished")
|
| 424 |
+
nav_info = gr.Markdown("")
|
| 425 |
+
|
| 426 |
+
# Move Admin accordion below so user_dd exists before wiring its updates
|
| 427 |
+
with gr.Accordion("Setup (admin) β define users and (re)deal assignments", open=False):
|
| 428 |
+
with gr.Row():
|
| 429 |
+
users_csv = gr.Textbox(value=",".join(DEFAULT_USERS),
|
| 430 |
+
label="Usernames (comma-separated)",
|
| 431 |
+
lines=2)
|
| 432 |
+
seed_num = gr.Number(value=DEFAULT_SEED, precision=0,
|
| 433 |
+
label="Assignment Seed (change & Apply)")
|
| 434 |
+
apply_btn = gr.Button("Apply / (Re)create Assignments", variant="secondary")
|
| 435 |
+
assign_info = gr.Markdown()
|
| 436 |
+
def apply_users(u_csv, seed):
|
| 437 |
+
usernames = [x.strip() for x in (u_csv or "").split(",") if x.strip()]
|
| 438 |
+
new_assign = create_assignments(usernames, int(seed or 0))
|
| 439 |
+
user_list = ", ".join(sorted(new_assign.keys()))
|
| 440 |
+
# summarize counts
|
| 441 |
+
counts = {u: len(v) for u, v in new_assign.items()}
|
| 442 |
+
table = "\n".join([f"- **{u}**: {counts[u]} instances" for u in sorted(counts)])
|
| 443 |
+
total_assigned = sum(counts.values())
|
| 444 |
+
total_available = len(INSTANCE_IDS)
|
| 445 |
+
unassigned = total_available - total_assigned
|
| 446 |
+
return gr.update(choices=available_users(), value=None), f"Assignments updated for {len(new_assign)} users (10 instances each):\n{table}\n\n**Summary:** {total_assigned} instances assigned, {unassigned} instances left unassigned out of {total_available} total."
|
| 447 |
+
# now that user_dd exists, reference it directly in outputs
|
| 448 |
+
apply_btn.click(apply_users, inputs=[users_csv, seed_num], outputs=[user_dd, assign_info])
|
| 449 |
+
|
| 450 |
+
# ---- Wiring functions ----
|
| 451 |
+
def _load_user(user: str):
|
| 452 |
+
if not user:
|
| 453 |
+
raise gr.Error("Please select a user.")
|
| 454 |
+
# Refresh assignments if changed by admin
|
| 455 |
+
init_or_load_assignments(DEFAULT_USERS, seed=DEFAULT_SEED)
|
| 456 |
+
_ensure_user_progress_struct(user)
|
| 457 |
+
|
| 458 |
+
# summary
|
| 459 |
+
txt, rows = summarize_user(user)
|
| 460 |
+
# initial pointer = next unfinished overall
|
| 461 |
+
inst, q_idx = next_unfinished(user)
|
| 462 |
+
assignments = _get_assignments()
|
| 463 |
+
if inst is None:
|
| 464 |
+
# user is done
|
| 465 |
+
return (
|
| 466 |
+
gr.update(choices=assignments.get(user, []), visible=True, value=None),
|
| 467 |
+
gr.update(visible=False),
|
| 468 |
+
"", "", "", "", "",
|
| 469 |
+
gr.update(value=None, interactive=True),
|
| 470 |
+
gr.update(value=None, interactive=False),
|
| 471 |
+
"", # note
|
| 472 |
+
txt, rows, "All assigned QA pairs are completed. π"
|
| 473 |
+
)
|
| 474 |
+
# populate content
|
| 475 |
+
q, a, f, im, header = get_payload(inst, q_idx)
|
| 476 |
+
n = len(DATA[inst]["qa_pairs"])
|
| 477 |
+
slider_update = gr.update(visible=(n > 0), minimum=1 if n > 0 else 1, maximum=n if n > 0 else 1, step=1, value=(q_idx+1 if n > 0 else 1))
|
| 478 |
+
nav = f"{assignments.get(user, []).index(inst)+1}/{len(assignments.get(user, []))} β’ Q{(q_idx+1) if n>0 else 0}/{n}"
|
| 479 |
+
return (
|
| 480 |
+
gr.update(choices=assignments.get(user, []), visible=True, value=inst),
|
| 481 |
+
slider_update,
|
| 482 |
+
f, im, f"**{header}**", f"**Q:** {q}", f"**A:** {a}",
|
| 483 |
+
gr.update(value=None, interactive=True),
|
| 484 |
+
gr.update(value=None, interactive=False),
|
| 485 |
+
"",
|
| 486 |
+
txt, rows, nav
|
| 487 |
+
)
|
| 488 |
+
|
| 489 |
+
load_btn.click(
|
| 490 |
+
_load_user,
|
| 491 |
+
inputs=[user_dd],
|
| 492 |
+
outputs=[inst_dd, q_slider, findings_tb, impressions_tb, header_md, question_md, answer_md,
|
| 493 |
+
relevant_radio, correct_radio, note_tb, progress_text, progress_table, nav_info]
|
| 494 |
+
)
|
| 495 |
+
|
| 496 |
+
def _inst_changed(user: str, inst: str):
|
| 497 |
+
if not user or not inst:
|
| 498 |
+
return gr.update(visible=False), "", "", "", "", "", gr.update(interactive=True), gr.update(interactive=False), "", ""
|
| 499 |
+
idx = first_unfinished_in_instance(user, inst)
|
| 500 |
+
q, a, f, im, header = get_payload(inst, idx)
|
| 501 |
+
n = len(DATA[inst]["qa_pairs"])
|
| 502 |
+
slider_update = gr.update(visible=(n > 0), minimum=1 if n > 0 else 1, maximum=n if n > 0 else 1, step=1, value=(idx+1 if n > 0 else 1))
|
| 503 |
+
assignments = _get_assignments()
|
| 504 |
+
return (
|
| 505 |
+
slider_update,
|
| 506 |
+
f, im, f"**{header}**", f"**Q:** {q}", f"**A:** {a}",
|
| 507 |
+
gr.update(value=None, interactive=True),
|
| 508 |
+
gr.update(value=None, interactive=False),
|
| 509 |
+
"",
|
| 510 |
+
f"{assignments.get(user, []).index(inst)+1}/{len(assignments.get(user, []))} β’ Q{(idx+1) if n>0 else 0}/{n}"
|
| 511 |
+
)
|
| 512 |
+
|
| 513 |
+
inst_dd.change(
|
| 514 |
+
_inst_changed,
|
| 515 |
+
inputs=[user_dd, inst_dd],
|
| 516 |
+
outputs=[q_slider, findings_tb, impressions_tb, header_md, question_md, answer_md,
|
| 517 |
+
relevant_radio, correct_radio, note_tb, nav_info]
|
| 518 |
+
)
|
| 519 |
+
|
| 520 |
+
def _q_changed(inst: str, q_no: int):
|
| 521 |
+
if not inst:
|
| 522 |
+
return "", "", "", ""
|
| 523 |
+
idx = int(q_no) - 1
|
| 524 |
+
q, a, f, im, header = get_payload(inst, idx)
|
| 525 |
+
return f"**{header}**", f"**Q:** {q}", f"**A:** {a}", ""
|
| 526 |
+
q_slider.change(_q_changed, inputs=[inst_dd, q_slider],
|
| 527 |
+
outputs=[header_md, question_md, answer_md, note_tb])
|
| 528 |
+
|
| 529 |
+
def _relevant_changed(rel_choice: Optional[str]):
|
| 530 |
+
if rel_choice == "β Relevant":
|
| 531 |
+
return gr.update(interactive=True)
|
| 532 |
+
else:
|
| 533 |
+
# reset and disable
|
| 534 |
+
return gr.update(value=None, interactive=False)
|
| 535 |
+
|
| 536 |
+
relevant_radio.change(_relevant_changed, inputs=[relevant_radio], outputs=[correct_radio])
|
| 537 |
+
|
| 538 |
+
def _save(user: str, inst: str, q_no: int, rel_choice: Optional[str], corr_choice: Optional[str], note: str):
|
| 539 |
+
if not (user and inst and q_no):
|
| 540 |
+
raise gr.Error("Missing user/instance/question selection.")
|
| 541 |
+
idx = int(q_no) - 1
|
| 542 |
+
msg = save_eval(user, inst, idx, rel_choice, corr_choice, note)
|
| 543 |
+
txt, rows = summarize_user(user)
|
| 544 |
+
return msg, txt, rows
|
| 545 |
+
|
| 546 |
+
save_btn.click(
|
| 547 |
+
_save,
|
| 548 |
+
inputs=[user_dd, inst_dd, q_slider, relevant_radio, correct_radio, note_tb],
|
| 549 |
+
outputs=[nav_info, progress_text, progress_table]
|
| 550 |
+
)
|
| 551 |
+
|
| 552 |
+
def _save_and_next(user: str, inst: str, q_no: int, rel_choice: Optional[str], corr_choice: Optional[str], note: str):
|
| 553 |
+
if not (user and inst and q_no):
|
| 554 |
+
raise gr.Error("Missing user/instance/question selection.")
|
| 555 |
+
idx = int(q_no) - 1
|
| 556 |
+
msg = save_eval(user, inst, idx, rel_choice, corr_choice, note)
|
| 557 |
+
# jump to next unfinished (global)
|
| 558 |
+
nxt_inst, nxt_idx = next_unfinished(user)
|
| 559 |
+
assignments = _get_assignments()
|
| 560 |
+
if nxt_inst is None:
|
| 561 |
+
txt, rows = summarize_user(user)
|
| 562 |
+
return (
|
| 563 |
+
gr.Dropdown.update(value=None),
|
| 564 |
+
gr.update(visible=False),
|
| 565 |
+
"", "", "",
|
| 566 |
+
"", "", # q/a
|
| 567 |
+
gr.update(value=None, interactive=True),
|
| 568 |
+
gr.update(value=None, interactive=False),
|
| 569 |
+
"",
|
| 570 |
+
f"{msg}\n\nAll assigned QA pairs are completed. π",
|
| 571 |
+
txt, rows
|
| 572 |
+
)
|
| 573 |
+
# else load that payload
|
| 574 |
+
q, a, f, im, header = get_payload(nxt_inst, nxt_idx)
|
| 575 |
+
n = len(DATA[nxt_inst]["qa_pairs"])
|
| 576 |
+
txt, rows = summarize_user(user)
|
| 577 |
+
slider_update = gr.update(visible=(n > 0), minimum=1 if n > 0 else 1, maximum=n if n > 0 else 1, step=1, value=(nxt_idx+1 if n > 0 else 1))
|
| 578 |
+
return (
|
| 579 |
+
gr.update(value=nxt_inst),
|
| 580 |
+
slider_update,
|
| 581 |
+
f, im, f"**{header}**",
|
| 582 |
+
f"**Q:** {q}", f"**A:** {a}",
|
| 583 |
+
gr.update(value=None, interactive=True),
|
| 584 |
+
gr.update(value=None, interactive=False),
|
| 585 |
+
"",
|
| 586 |
+
msg,
|
| 587 |
+
txt, rows
|
| 588 |
+
)
|
| 589 |
+
|
| 590 |
+
save_next_btn.click(
|
| 591 |
+
_save_and_next,
|
| 592 |
+
inputs=[user_dd, inst_dd, q_slider, relevant_radio, correct_radio, note_tb],
|
| 593 |
+
outputs=[inst_dd, q_slider, findings_tb, impressions_tb, header_md, question_md, answer_md,
|
| 594 |
+
relevant_radio, correct_radio, note_tb, nav_info, progress_text, progress_table]
|
| 595 |
+
)
|
| 596 |
+
|
| 597 |
+
def _skip_to_next(user: str):
|
| 598 |
+
if not user:
|
| 599 |
+
raise gr.Error("Please select a user.")
|
| 600 |
+
inst, idx = next_unfinished(user)
|
| 601 |
+
assignments = _get_assignments()
|
| 602 |
+
if inst is None:
|
| 603 |
+
txt, rows = summarize_user(user)
|
| 604 |
+
return (
|
| 605 |
+
gr.update(value=None),
|
| 606 |
+
gr.update(visible=False),
|
| 607 |
+
"", "", "",
|
| 608 |
+
"", "",
|
| 609 |
+
gr.update(value=None, interactive=True),
|
| 610 |
+
gr.update(value=None, interactive=False),
|
| 611 |
+
"",
|
| 612 |
+
"All assigned QA pairs are completed. π",
|
| 613 |
+
txt, rows
|
| 614 |
+
)
|
| 615 |
+
q, a, f, im, header = get_payload(inst, idx)
|
| 616 |
+
n = len(DATA[inst]["qa_pairs"])
|
| 617 |
+
txt, rows = summarize_user(user)
|
| 618 |
+
slider_update = gr.update(visible=(n > 0), minimum=1 if n > 0 else 1, maximum=n if n > 0 else 1, step=1, value=(idx+1 if n > 0 else 1))
|
| 619 |
+
return (
|
| 620 |
+
gr.update(value=inst),
|
| 621 |
+
slider_update,
|
| 622 |
+
f, im, f"**{header}**",
|
| 623 |
+
f"**Q:** {q}", f"**A:** {a}",
|
| 624 |
+
gr.update(value=None, interactive=True),
|
| 625 |
+
gr.update(value=None, interactive=False),
|
| 626 |
+
"",
|
| 627 |
+
"Jumped to next unfinished.",
|
| 628 |
+
txt, rows
|
| 629 |
+
)
|
| 630 |
+
|
| 631 |
+
skip_btn.click(
|
| 632 |
+
_skip_to_next,
|
| 633 |
+
inputs=[user_dd],
|
| 634 |
+
outputs=[inst_dd, q_slider, findings_tb, impressions_tb, header_md, question_md, answer_md,
|
| 635 |
+
relevant_radio, correct_radio, note_tb, nav_info, progress_text, progress_table]
|
| 636 |
+
)
|
| 637 |
+
|
| 638 |
+
def _export(user: str):
|
| 639 |
+
msg = export_user_results(user)
|
| 640 |
+
return msg
|
| 641 |
+
export_btn.click(_export, inputs=[user_dd], outputs=[nav_info])
|
| 642 |
+
|
| 643 |
+
if __name__ == "__main__":
|
| 644 |
+
# server_name '0.0.0.0' allows remote access if hosted; keep default port
|
| 645 |
+
demo.launch(share=True)
|