Commit ·
ed1313f
1
Parent(s): 7b08c83
RTS Human Evaluation
Browse files- .gitattributes +5 -0
- app.py +24 -24
- data/gt/train_nitze_016.png +3 -0
- data/gt/train_nitze_106.png +3 -0
- data/gt/train_yg_078.png +3 -0
- data/gt/train_yg_300.png +3 -0
- data/gt/train_yg_304.png +3 -0
- data/sr/train_nitze_016.png +3 -0
- data/sr/train_nitze_106.png +3 -0
- data/sr/train_yg_078.png +3 -0
- data/sr/train_yg_300.png +3 -0
- data/sr/train_yg_304.png +3 -0
- requirements.txt +4 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
*.jpeg filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
*.tif filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
*.tiff filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
*.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
*.jpg filter=lfs diff=lfs merge=lfs -text
|
app.py
CHANGED
|
@@ -15,9 +15,9 @@ from filelock import FileLock
|
|
| 15 |
# Configuration
|
| 16 |
# ----------------------
|
| 17 |
|
| 18 |
-
SR_DIR = os.environ.get("SR_DIR", "
|
| 19 |
-
GT_DIR = os.environ.get("GT_DIR", "
|
| 20 |
-
RESULTS_DIR = os.environ.get("RESULTS_DIR", "
|
| 21 |
PROGRESS_PATH = os.path.join(RESULTS_DIR, "progress.json")
|
| 22 |
ALL_RESULTS_JSONL = os.path.join(RESULTS_DIR, "all_results.jsonl")
|
| 23 |
SAVE_PII = True # set False to only store hashed user_id
|
|
@@ -213,8 +213,8 @@ def load_dataset(sr_dir: str, gt_dir: str) -> List[Sample]:
|
|
| 213 |
# Progress & results I/O
|
| 214 |
# ----------------------
|
| 215 |
|
| 216 |
-
def hash_user_id(name: str,
|
| 217 |
-
norm = (name or "").strip().lower() + "|" + (
|
| 218 |
return hashlib.sha256(norm.encode("utf-8")).hexdigest()[:16]
|
| 219 |
|
| 220 |
|
|
@@ -363,14 +363,14 @@ def step_display_index(step: int, detect_choice: str | None) -> tuple[int, int]:
|
|
| 363 |
# Gradio callbacks (core)
|
| 364 |
# ----------------------
|
| 365 |
|
| 366 |
-
def start_or_resume(name: str,
|
| 367 |
-
if not name or not
|
| 368 |
-
raise gr.Error("Please enter your name and
|
| 369 |
|
| 370 |
ensure_paths()
|
| 371 |
samples = load_dataset(SR_DIR, GT_DIR)
|
| 372 |
|
| 373 |
-
uid = hash_user_id(name,
|
| 374 |
progress = load_progress()
|
| 375 |
if uid not in progress:
|
| 376 |
progress[uid] = {"seen": []}
|
|
@@ -429,7 +429,7 @@ def update_overlay(test_img: Image.Image, selected_tiles: List[str] | None):
|
|
| 429 |
|
| 430 |
def _save_record_and_progress(
|
| 431 |
name: str,
|
| 432 |
-
|
| 433 |
uid: str,
|
| 434 |
samples: List[Sample],
|
| 435 |
user_seen: List[str],
|
|
@@ -442,8 +442,8 @@ def _save_record_and_progress(
|
|
| 442 |
reasoning: str,
|
| 443 |
):
|
| 444 |
"""Common save logic used by both 'finish' and 'next image' paths."""
|
| 445 |
-
if not name or not
|
| 446 |
-
raise gr.Error("Please enter your name and
|
| 447 |
if detection_choice not in {"Yes", "No", "Unsure"}:
|
| 448 |
detection_choice = "Unsure"
|
| 449 |
valid_clarity = {
|
|
@@ -464,7 +464,7 @@ def _save_record_and_progress(
|
|
| 464 |
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
| 465 |
"user_id": uid,
|
| 466 |
"name": name if SAVE_PII else None,
|
| 467 |
-
"
|
| 468 |
"sample_id": sample.sample_id,
|
| 469 |
"sr_path": sample.sr_path,
|
| 470 |
"gt_path": sample.gt_path,
|
|
@@ -499,7 +499,7 @@ def _save_record_and_progress(
|
|
| 499 |
|
| 500 |
def submit_finish(
|
| 501 |
name: str,
|
| 502 |
-
|
| 503 |
uid: str,
|
| 504 |
samples: List[Sample],
|
| 505 |
user_seen: List[str],
|
|
@@ -513,7 +513,7 @@ def submit_finish(
|
|
| 513 |
):
|
| 514 |
"""Save and show Thank You (no auto-next)."""
|
| 515 |
progress = _save_record_and_progress(
|
| 516 |
-
name,
|
| 517 |
detection_choice, confidence, selected_tiles, clarity_choice, reasoning
|
| 518 |
)
|
| 519 |
|
|
@@ -549,7 +549,7 @@ def submit_finish(
|
|
| 549 |
|
| 550 |
def submit_next_image(
|
| 551 |
name: str,
|
| 552 |
-
|
| 553 |
uid: str,
|
| 554 |
samples: List[Sample],
|
| 555 |
user_seen: List[str],
|
|
@@ -563,7 +563,7 @@ def submit_next_image(
|
|
| 563 |
):
|
| 564 |
"""Save and immediately load the next sample."""
|
| 565 |
progress = _save_record_and_progress(
|
| 566 |
-
name,
|
| 567 |
detection_choice, confidence, selected_tiles, clarity_choice, reasoning
|
| 568 |
)
|
| 569 |
# NEW — mark global done & release current reservation
|
|
@@ -728,7 +728,7 @@ with gr.Blocks(title="RTS Human Evaluation", theme=gr.themes.Soft()) as demo:
|
|
| 728 |
# Retrogressive Thaw Slump (RTS) Human Evaluation
|
| 729 |
|
| 730 |
**Instructions:**
|
| 731 |
-
1. Enter your name and address, then click **Start / Resume**.
|
| 732 |
2. Use **Next** and **Back** to go through the questions.
|
| 733 |
3. On the **last step**, choose **Submit & Next Image** to continue, or **Submit & Finish** to end and see a Thank You screen (with **Start Again**).
|
| 734 |
"""
|
|
@@ -745,7 +745,7 @@ with gr.Blocks(title="RTS Human Evaluation", theme=gr.themes.Soft()) as demo:
|
|
| 745 |
with gr.Group() as start_group:
|
| 746 |
with gr.Row():
|
| 747 |
name = gr.Textbox(label="Full name", placeholder="Jane Doe", autofocus=True)
|
| 748 |
-
|
| 749 |
start_btn = gr.Button("Start / Resume", variant="primary")
|
| 750 |
status = gr.Markdown("\n")
|
| 751 |
|
|
@@ -823,7 +823,7 @@ with gr.Blocks(title="RTS Human Evaluation", theme=gr.themes.Soft()) as demo:
|
|
| 823 |
# --- Wiring ---
|
| 824 |
start_event = start_btn.click(
|
| 825 |
start_or_resume,
|
| 826 |
-
inputs=[name,
|
| 827 |
outputs=[
|
| 828 |
state_uid, state_samples, state_seen, state_idx, state_a_is_sr,
|
| 829 |
test_img, overlay_img, image_a, image_b, status, your_jsonl_path, eval_panel, tiles, state_step, intro_md, start_group
|
|
@@ -867,7 +867,7 @@ with gr.Blocks(title="RTS Human Evaluation", theme=gr.themes.Soft()) as demo:
|
|
| 867 |
submit_finish,
|
| 868 |
inputs=[
|
| 869 |
name,
|
| 870 |
-
|
| 871 |
state_uid,
|
| 872 |
state_samples,
|
| 873 |
state_seen,
|
|
@@ -906,7 +906,7 @@ with gr.Blocks(title="RTS Human Evaluation", theme=gr.themes.Soft()) as demo:
|
|
| 906 |
submit_next_image,
|
| 907 |
inputs=[
|
| 908 |
name,
|
| 909 |
-
|
| 910 |
state_uid,
|
| 911 |
state_samples,
|
| 912 |
state_seen,
|
|
@@ -947,10 +947,10 @@ with gr.Blocks(title="RTS Human Evaluation", theme=gr.themes.Soft()) as demo:
|
|
| 947 |
outputs=[step_status, group_detection, group_confidence, group_tiles, group_clarity, group_reasoning, back_btn, next_btn],
|
| 948 |
)
|
| 949 |
|
| 950 |
-
# Restart: start again using current name/
|
| 951 |
restart_event = restart_btn.click(
|
| 952 |
start_or_resume,
|
| 953 |
-
inputs=[name,
|
| 954 |
outputs=[
|
| 955 |
state_uid, state_samples, state_seen, state_idx, state_a_is_sr,
|
| 956 |
test_img, overlay_img, image_a, image_b, status, your_jsonl_path, eval_panel, tiles, state_step, intro_md, start_group
|
|
|
|
| 15 |
# Configuration
|
| 16 |
# ----------------------
|
| 17 |
|
| 18 |
+
SR_DIR = os.environ.get("SR_DIR", "data/sr")
|
| 19 |
+
GT_DIR = os.environ.get("GT_DIR", "data/gt")
|
| 20 |
+
RESULTS_DIR = os.environ.get("RESULTS_DIR", "results")
|
| 21 |
PROGRESS_PATH = os.path.join(RESULTS_DIR, "progress.json")
|
| 22 |
ALL_RESULTS_JSONL = os.path.join(RESULTS_DIR, "all_results.jsonl")
|
| 23 |
SAVE_PII = True # set False to only store hashed user_id
|
|
|
|
| 213 |
# Progress & results I/O
|
| 214 |
# ----------------------
|
| 215 |
|
| 216 |
+
def hash_user_id(name: str, email: str) -> str:
|
| 217 |
+
norm = (name or "").strip().lower() + "|" + (email or "").strip().lower()
|
| 218 |
return hashlib.sha256(norm.encode("utf-8")).hexdigest()[:16]
|
| 219 |
|
| 220 |
|
|
|
|
| 363 |
# Gradio callbacks (core)
|
| 364 |
# ----------------------
|
| 365 |
|
| 366 |
+
def start_or_resume(name: str, email: str):
|
| 367 |
+
if not name or not email:
|
| 368 |
+
raise gr.Error("Please enter your name and email to begin.")
|
| 369 |
|
| 370 |
ensure_paths()
|
| 371 |
samples = load_dataset(SR_DIR, GT_DIR)
|
| 372 |
|
| 373 |
+
uid = hash_user_id(name, email)
|
| 374 |
progress = load_progress()
|
| 375 |
if uid not in progress:
|
| 376 |
progress[uid] = {"seen": []}
|
|
|
|
| 429 |
|
| 430 |
def _save_record_and_progress(
|
| 431 |
name: str,
|
| 432 |
+
email: str,
|
| 433 |
uid: str,
|
| 434 |
samples: List[Sample],
|
| 435 |
user_seen: List[str],
|
|
|
|
| 442 |
reasoning: str,
|
| 443 |
):
|
| 444 |
"""Common save logic used by both 'finish' and 'next image' paths."""
|
| 445 |
+
if not name or not email:
|
| 446 |
+
raise gr.Error("Please enter your name and email.")
|
| 447 |
if detection_choice not in {"Yes", "No", "Unsure"}:
|
| 448 |
detection_choice = "Unsure"
|
| 449 |
valid_clarity = {
|
|
|
|
| 464 |
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
| 465 |
"user_id": uid,
|
| 466 |
"name": name if SAVE_PII else None,
|
| 467 |
+
"email": email if SAVE_PII else None,
|
| 468 |
"sample_id": sample.sample_id,
|
| 469 |
"sr_path": sample.sr_path,
|
| 470 |
"gt_path": sample.gt_path,
|
|
|
|
| 499 |
|
| 500 |
def submit_finish(
|
| 501 |
name: str,
|
| 502 |
+
email: str,
|
| 503 |
uid: str,
|
| 504 |
samples: List[Sample],
|
| 505 |
user_seen: List[str],
|
|
|
|
| 513 |
):
|
| 514 |
"""Save and show Thank You (no auto-next)."""
|
| 515 |
progress = _save_record_and_progress(
|
| 516 |
+
name, email, uid, samples, user_seen, idx, a_is_sr,
|
| 517 |
detection_choice, confidence, selected_tiles, clarity_choice, reasoning
|
| 518 |
)
|
| 519 |
|
|
|
|
| 549 |
|
| 550 |
def submit_next_image(
|
| 551 |
name: str,
|
| 552 |
+
email: str,
|
| 553 |
uid: str,
|
| 554 |
samples: List[Sample],
|
| 555 |
user_seen: List[str],
|
|
|
|
| 563 |
):
|
| 564 |
"""Save and immediately load the next sample."""
|
| 565 |
progress = _save_record_and_progress(
|
| 566 |
+
name, email, uid, samples, user_seen, idx, a_is_sr,
|
| 567 |
detection_choice, confidence, selected_tiles, clarity_choice, reasoning
|
| 568 |
)
|
| 569 |
# NEW — mark global done & release current reservation
|
|
|
|
| 728 |
# Retrogressive Thaw Slump (RTS) Human Evaluation
|
| 729 |
|
| 730 |
**Instructions:**
|
| 731 |
+
1. Enter your name and email address, then click **Start / Resume**.
|
| 732 |
2. Use **Next** and **Back** to go through the questions.
|
| 733 |
3. On the **last step**, choose **Submit & Next Image** to continue, or **Submit & Finish** to end and see a Thank You screen (with **Start Again**).
|
| 734 |
"""
|
|
|
|
| 745 |
with gr.Group() as start_group:
|
| 746 |
with gr.Row():
|
| 747 |
name = gr.Textbox(label="Full name", placeholder="Jane Doe", autofocus=True)
|
| 748 |
+
email = gr.Textbox(label="Email address", placeholder="jane@example.com")
|
| 749 |
start_btn = gr.Button("Start / Resume", variant="primary")
|
| 750 |
status = gr.Markdown("\n")
|
| 751 |
|
|
|
|
| 823 |
# --- Wiring ---
|
| 824 |
start_event = start_btn.click(
|
| 825 |
start_or_resume,
|
| 826 |
+
inputs=[name, email],
|
| 827 |
outputs=[
|
| 828 |
state_uid, state_samples, state_seen, state_idx, state_a_is_sr,
|
| 829 |
test_img, overlay_img, image_a, image_b, status, your_jsonl_path, eval_panel, tiles, state_step, intro_md, start_group
|
|
|
|
| 867 |
submit_finish,
|
| 868 |
inputs=[
|
| 869 |
name,
|
| 870 |
+
email,
|
| 871 |
state_uid,
|
| 872 |
state_samples,
|
| 873 |
state_seen,
|
|
|
|
| 906 |
submit_next_image,
|
| 907 |
inputs=[
|
| 908 |
name,
|
| 909 |
+
email,
|
| 910 |
state_uid,
|
| 911 |
state_samples,
|
| 912 |
state_seen,
|
|
|
|
| 947 |
outputs=[step_status, group_detection, group_confidence, group_tiles, group_clarity, group_reasoning, back_btn, next_btn],
|
| 948 |
)
|
| 949 |
|
| 950 |
+
# Restart: start again using current name/email
|
| 951 |
restart_event = restart_btn.click(
|
| 952 |
start_or_resume,
|
| 953 |
+
inputs=[name, email],
|
| 954 |
outputs=[
|
| 955 |
state_uid, state_samples, state_seen, state_idx, state_a_is_sr,
|
| 956 |
test_img, overlay_img, image_a, image_b, status, your_jsonl_path, eval_panel, tiles, state_step, intro_md, start_group
|
data/gt/train_nitze_016.png
ADDED
|
Git LFS Details
|
data/gt/train_nitze_106.png
ADDED
|
Git LFS Details
|
data/gt/train_yg_078.png
ADDED
|
Git LFS Details
|
data/gt/train_yg_300.png
ADDED
|
Git LFS Details
|
data/gt/train_yg_304.png
ADDED
|
Git LFS Details
|
data/sr/train_nitze_016.png
ADDED
|
Git LFS Details
|
data/sr/train_nitze_106.png
ADDED
|
Git LFS Details
|
data/sr/train_yg_078.png
ADDED
|
Git LFS Details
|
data/sr/train_yg_300.png
ADDED
|
Git LFS Details
|
data/sr/train_yg_304.png
ADDED
|
Git LFS Details
|
requirements.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
gradio>=4.29
|
| 2 |
+
pillow
|
| 3 |
+
numpy
|
| 4 |
+
filelock
|