import os, io, csv, json, random from datetime import datetime import gradio as gr from huggingface_hub import HfApi, hf_hub_download # -------------------- Config -------------------- REPO_ID = os.getenv("RESULTS_REPO", "sgtlim/videoeval_results") # 업로드한 리포와 일치 HF_TOKEN = os.getenv("HF_TOKEN") RESULTS_FILE = "results.csv" TOTAL_PER_PARTICIPANT = 30 # 목표 평가 개수(세션 기준) # videos.json 예시: {"url": "...mp4", "id": "BodyWeightSquats__XXXX.mp4", "action": "BodyWeightSquats"} with open("videos.json", "r", encoding="utf-8") as f: V = json.load(f) api = HfApi() # 교수님 지침(그대로, 굵게 처리 포함) INSTRUCTION_MD = """ **Task:** You will watch a series of **AI-generated videos**. For each video, your job is to rate how well the person’s action in the AI-generated video matches the action specified as "**expected action**". Some things to keep in mind: - The generated video should **capture** the expected action **throughout the video**. - Try to **focus only** on the expected action and do **not** judge **video quality**, **attractiveness**, **background**, **camera motion**, or **objects**. - You will be **paid** once **all the videos are viewed and rated**. """ # -------------------- Helper funcs -------------------- def _load_eval_counts(): """ Hugging Face dataset의 results.csv를 읽어 video_id별 평가 개수(dict)를 반환. 없으면 0으로 초기화. """ # 모든 id를 0으로 초기화 counts = {} for v in V: vid = _get_video_id(v) counts[vid] = 0 b = _read_csv_bytes() if not b: return counts s = io.StringIO(b.decode("utf-8", errors="ignore")) r = csv.reader(s) rows = list(r) if not rows: return counts # 헤더 파악 header = rows[0] body = rows[1:] if header and ("video_id" in header or "overall" in header) else rows vid_col = None if header and "video_id" in header: vid_col = header.index("video_id") for row in body: try: vid = row[vid_col] if vid_col is not None else row[2] # 기본 포맷: ts, pid, video_id, overall, notes if vid in counts: counts[vid] += 1 except Exception: continue return counts def _get_video_id(v: dict) -> str: if "id" in v and v["id"]: return v["id"] # id가 없으면 URL 파일명으로 대체 return os.path.basename(v.get("url", "")) def _read_csv_bytes(): try: p = hf_hub_download( repo_id=REPO_ID, filename=RESULTS_FILE, repo_type="dataset", token=HF_TOKEN, local_dir="/tmp", local_dir_use_symlinks=False ) return open(p, "rb").read() except Exception: return None def _append(old_bytes, row): s = io.StringIO() w = csv.writer(s) if not old_bytes: # ✅ 새 헤더 w.writerow(["ts_iso", "participant_id", "video_id", "overall", "notes"]) else: s.write(old_bytes.decode("utf-8", errors="ignore")) w.writerow(row) return s.getvalue().encode("utf-8") # def push(participant_id, action_name, score, notes=""): # if not participant_id or not participant_id.strip(): # return gr.update(visible=True, value="❗ Please enter your Participant ID before proceeding.") # if not action_name or score is None: # return gr.update(visible=True, value="❗ Fill out all fields.") # old = _read_csv_bytes() # row = [ # datetime.utcnow().isoformat(), # participant_id.strip(), # action_name, # float(score), # notes or "" # ] # newb = _append(old, row) # api.upload_file( # path_or_fileobj=io.BytesIO(newb), # path_in_repo=RESULTS_FILE, # repo_id=REPO_ID, # repo_type="dataset", # token=HF_TOKEN, # commit_message="append" # ) # return gr.update(visible=True, value=f"✅ Saved for {action_name}.") def push(participant_id, video_id, score, notes=""): if not participant_id or not participant_id.strip(): return gr.update(visible=True, value="❗ Please enter your Participant ID before proceeding.") if not video_id or score is None: return gr.update(visible=True, value="❗ Fill out all fields.") try: old = _read_csv_bytes() row = [ datetime.utcnow().isoformat(), participant_id.strip(), video_id, # ✅ action 대신 video_id 저장 float(score), # overall notes or "" ] newb = _append(old, row) if not REPO_ID: return gr.update(visible=True, value="❗ RESULTS_REPO is not set.") if not HF_TOKEN: return gr.update(visible=True, value="❗ HF_TOKEN is missing. Set a write token for the dataset repo.") api.upload_file( path_or_fileobj=io.BytesIO(newb), path_in_repo=RESULTS_FILE, repo_id=REPO_ID, repo_type="dataset", token=HF_TOKEN, commit_message="append" ) return gr.update(visible=True, value=f"✅ Saved for {video_id}.") except Exception as e: return gr.update( visible=True, value=f"❌ Save failed: {type(e).__name__}: {e}\n" f"- Check HF_TOKEN permission\n- Check REPO_ID\n- Create dataset repo if missing" ) def _extract_action(v): if "action" in v and v["action"]: return v["action"] raw = v.get("id", "") return raw.split("__")[0].split(".")[0] def pick_one(): v = random.choice(V) return v["url"], _extract_action(v) def _progress_html(done, total): pct = int(100 * done / max(1, total)) return f"""
{done} / {total}
""" def _build_order_with_anchor(total:int, anchor_idx:int, repeats:int, pool_size:int, min_gap:int=1): """ total: TOTAL_PER_PARTICIPANT (e.g., 30) anchor_idx: index of the anchor video in V (0 for first item) repeats: how many times to show anchor (e.g., 5) pool_size: len(V) min_gap: 최소 간격(인접 금지 => 1) return: list of indices (length=total) """ assert repeats <= total, "repeats must be <= total" assert pool_size >= 1, "videos pool must be non-empty" # 1) 다른 비디오 25개(중복 없이) 뽑기 others_needed = total - repeats # anchor를 제외한 후보 인덱스 candidates = list(range(1, pool_size)) if anchor_idx == 0 else [i for i in range(pool_size) if i != anchor_idx] if len(candidates) < others_needed: raise ValueError("Not enough unique non-anchor videos to fill the schedule without duplication.") others = random.sample(candidates, k=others_needed) # 2) 기본 시퀀스(others)를 무작위로 섞기 random.shuffle(others) # 3) 앵커를 min_gap를 만족하도록 삽입할 위치 선정 # 30개를 5구간으로 나눠, 각 구간 내에서 충돌 덜 나게 배치 # (간단하고 안정적인 방식) seq = others[:] # 길이=25 anchor_positions = [] segment = total // repeats # 30//5 = 6 for k in range(repeats): # 각 구간 [k*segment, (k+1)*segment) 안에서 후보 위치를 고름 lo = k * segment hi = (k + 1) * segment if k < repeats - 1 else total # 마지막은 끝까지 # 경계 내 임의 오프셋 선택 (여유를 두고 충돌을 피함) candidate_pos = random.randrange(lo, hi) # 인접 금지 보정: 이미 배정된 anchor 위치들과의 거리가 min_gap 이상 되도록 조정 # 필요 시 좌우로 근접한 빈 슬롯 탐색 def ok(pos): return all(abs(pos - p) >= min_gap + 1 for p in anchor_positions) # 연속금지 => 거리 >= 2 # 근방 탐색 폭 found = None for delta in range(0, segment): # 구간 크기 내에서 탐색 # 좌/우 번갈아가며 후보 시도 for sign in (+1, -1): pos = candidate_pos + sign * delta if 0 <= pos < total and ok(pos): found = pos break if found is not None: break if found is None: # 최후: 0..total-1 범위에서 아무 데나 충돌 없는 곳 찾기 for pos in range(total): if ok(pos): found = pos break if found is None: raise RuntimeError("Failed to place anchor without adjacency. Try different strategy or loosen min_gap.") anchor_positions.append(found) # 4) others를 기반으로 길이 total의 빈 시퀀스를 만들고 anchor를 주입 # 우선 빈 리스트를 만들고 anchor 위치를 채운 후, 나머지를 others로 채움 result = [None] * total for pos in anchor_positions: result[pos] = anchor_idx # others 포인터 j = 0 for i in range(total): if result[i] is None: result[i] = others[j] j += 1 # 안전체크 assert len(result) == total # 인접 anchor 없는지 확인 for i in range(1, total): assert not (result[i] == anchor_idx and result[i-1] == anchor_idx), "Adjacent anchors found." # anchor 개수 확인 assert sum(1 for x in result if x == anchor_idx) == repeats, "Anchor count mismatch." return result # -------------------- Example videos (download to local cache) -------------------- EXAMPLES = { "BodyWeightSquats": { "real": "examples/BodyWeightSquats_real.mp4", "bad": "examples/BodyWeightSquats_bad.mp4", }, "WallPushUps": { "real": "examples/WallPushUps_real.mp4", "bad": "examples/WallPushUps_bad.mp4", }, } EX_CACHE = {} for cls, files in EXAMPLES.items(): EX_CACHE[cls] = {"real": None, "bad": None} for kind, fname in files.items(): try: EX_CACHE[cls][kind] = hf_hub_download( repo_id=REPO_ID, filename=fname, repo_type="dataset", token=HF_TOKEN, local_dir="/tmp", local_dir_use_symlinks=False, ) except Exception as e: print(f"[WARN] example missing: {cls} {kind} -> {fname}: {e}") GLOBAL_CSS = """ /* ===== 공통 변수 투명화 (v3/v4 둘다) ===== */ :root, .gradio-container { --body-background-fill: transparent !important; --background-fill-primary: transparent !important; --background-fill-secondary: transparent !important; --block-background-fill: transparent !important; --block-border-color: transparent !important; --panel-background-fill: transparent !important; --panel-border-color: transparent !important; --section-header-background-fill: transparent !important; --shadow-drop: 0 0 0 rgba(0,0,0,0) !important; --shadow-spread: 0 0 0 rgba(0,0,0,0) !important; } /* ===== v4(Tailwind 기반)에서 자주 쓰이는 배경/테두리/그림자 제거 ===== */ .gradio-container .bg-white, .gradio-container .bg-gray-50, .gradio-container .bg-gray-100, .gradio-container .bg-slate-50, .gradio-container .bg-neutral-50, .gradio-container .bg-secondary, .gradio-container .border, .gradio-container .shadow, .gradio-container .shadow-sm, .gradio-container .shadow-md, .gradio-container .ring-1, .gradio-container .ring, .gradio-container .gr-card, .gradio-container .prose > *:where(hr) { background: transparent !important; box-shadow: none !important; border-color: transparent !important; } /* ===== v3 컴포넌트 계열 ===== */ .gradio-container .gr-panel, .gradio-container .gr-group, .gradio-container .gr-box, .gradio-container .gr-row, .gradio-container .gr-column, .gradio-container .gr-accordion, .gradio-container .gr-block, .gradio-container .gr-form, .gradio-container .gr-tabs, .gradio-container .gr-tabitem, .gradio-container .gr-section-header { background: transparent !important; box-shadow: none !important; border: none !important; } /* 구분선/헤더 바 제거 */ .gradio-container hr, .gradio-container .gr-divider, .gradio-container .gr-accordion .label { background: transparent !important; border: none !important; box-shadow: none !important; } /* 바깥쪽 페이지 배경도 강제로 투명/흰색으로 */ html, body, .gradio-container { background: transparent !important; } /* 기존 CSS 아래에 추가 */ #eval [class*="bg-"], #eval [class*="border"], #eval [class*="shadow"], #eval .gr-panel, #eval .gr-group, #eval .gr-box, #eval .gr-row, #eval .gr-column, #eval .gr-block, #eval .gr-form, #eval .gr-section-header, #eval .gr-accordion { background: transparent !important; border-color: transparent !important; box-shadow: none !important; } #eval .gr-form, #eval .gr-panel { background: transparent !important; box-shadow:none !important; border:none !important; } """ # -------------------- UI -------------------- with gr.Blocks(fill_height=True, css=GLOBAL_CSS) as demo: order_state = gr.State(value=[]) # v4에서는 value= 권장 ptr_state = gr.State(value=0) cur_video_id = gr.State(value="") # ------------------ PAGE 1: Intro + Examples ------------------ page_intro = gr.Group(visible=True) with page_intro: gr.Markdown("## 🎯 Action Consistency Human Evaluation") gr.Markdown(INSTRUCTION_MD) # Examples: Squats with gr.Group(): gr.Markdown("### Examples: BodyWeightSquats") with gr.Row(): with gr.Column(): gr.Markdown("**Expected depiction of action**") gr.Video(value=EX_CACHE["BodyWeightSquats"]["real"], height=240, autoplay=False) with gr.Column(): gr.Markdown("**Poorly generated action**") gr.Video(value=EX_CACHE["BodyWeightSquats"]["bad"], height=240, autoplay=False) if not (EX_CACHE["BodyWeightSquats"]["real"] and EX_CACHE["BodyWeightSquats"]["bad"]): gr.Markdown("> ⚠️ Upload `examples/BodyWeightSquats_real.mp4` and `_bad.mp4` to show both samples.") # Examples: WallPushUps with gr.Group(): gr.Markdown("### Examples: WallPushUps") with gr.Row(): with gr.Column(): gr.Markdown("**Expected depiction of action**") gr.Video(value=EX_CACHE["WallPushUps"]["real"], height=240, autoplay=False) with gr.Column(): gr.Markdown("**Poorly generated action**") gr.Video(value=EX_CACHE["WallPushUps"]["bad"], height=240, autoplay=False) if not (EX_CACHE["WallPushUps"]["real"] and EX_CACHE["WallPushUps"]["bad"]): gr.Markdown("> ⚠️ Upload `examples/WallPushUps_real.mp4` and `_bad.mp4` to show both samples.") understood = gr.Checkbox(label="I have read and understand the task.", value=False) start_btn = gr.Button("Yes, start", variant="secondary", interactive=False) def _toggle_start(checked: bool): return gr.update(interactive=checked, variant=("primary" if checked else "secondary")) understood.change(_toggle_start, inputs=understood, outputs=start_btn) # ------------------ PAGE 2: Evaluation ------------------ page_eval = gr.Group(visible=False, elem_id="eval") with page_eval: # PID 입력 with gr.Row(): pid = gr.Textbox(label="Participant ID (required)", placeholder="e.g., Youngsun-2025/10/01") # 지침(원문) + 비디오 + 진행바 / 오른쪽에 슬라이더 + Save&Next with gr.Row(equal_height=True): with gr.Column(scale=1): gr.Markdown(INSTRUCTION_MD) # 교수님 문구 그대로 video = gr.Video(label="Video", height=360) progress = gr.HTML(_progress_html(0, TOTAL_PER_PARTICIPANT)) with gr.Column(scale=1): action_tb = gr.Textbox(label="Expected action", interactive=False) score = gr.Slider(minimum=0.0, maximum=10.0, step=0.1, value=5.0, label="Action Consistency (0.0 (Worst) - 10.0 (Best))") save_next = gr.Button("💾 Save & Next ▶", variant="secondary", interactive=False) status = gr.Markdown(visible=False) done_state = gr.State(0) # PID 입력에 따라 Save&Next 토글 def _toggle_by_pid(pid_text: str): enabled = bool(pid_text and pid_text.strip()) return gr.update(interactive=enabled, variant=("primary" if enabled else "secondary")) pid.change(_toggle_by_pid, inputs=pid, outputs=save_next) # -------- 페이지 전환 & 첫 로드 -------- ANCHOR_IDX = 0 # videos.json의 맨 첫 비디오 ANCHOR_REPEATS = 5 # 앵커 5회 MIN_GAP = 1 # 앵커 연속 금지(인접 금지) def _build_order_least_first_with_anchor(total:int, anchor_idx:int, repeats:int, min_gap:int=1): """ - results.csv를 읽어 video_id별 카운트를 계산 - 앵커(첫 비디오) 5회 포함, 연속 금지 - 나머지는 '가장 적게 평가된 순'으로 중복 없이 채움 """ assert repeats <= total N = len(V) assert N >= 1 # 0) id 매핑 def vid_of(i): return _get_video_id(V[i]) # 1) 현재 누적 카운트 로드 counts = _load_eval_counts() # 2) 앵커 제외 후보(중복 없이) 정렬: 카운트 오름차순, 동률은 랜덤 셔플 anchor_vid = vid_of(anchor_idx) candidates = [i for i in range(N) if i != anchor_idx] # 동률 랜덤화를 위해 일단 셔플 random.shuffle(candidates) candidates.sort(key=lambda i: counts.get(vid_of(i), 0)) others_needed = total - repeats if len(candidates) < others_needed: raise ValueError("Not enough unique non-anchor videos to fill the schedule without duplication.") others = candidates[:others_needed] # 중복 없이 선택 # 3) others를 베이스 시퀀스로(랜덤 살짝 섞기) random.shuffle(others) # 4) 앵커를 구간 배치(연속 금지) seq = [None] * total segment = total // repeats if repeats > 0 else total anchor_positions = [] for k in range(repeats): lo = k * segment hi = (k + 1) * segment if k < repeats - 1 else total cand = random.randrange(lo, hi) def ok(pos): return all(abs(pos - p) >= (min_gap + 1) for p in anchor_positions) found = None for d in range(0, max(1, segment)): for sgn in (+1, -1): pos = cand + sgn * d if 0 <= pos < total and ok(pos): found = pos break if found is not None: break if found is None: # 마지막 수단: 전체 탐색 for pos in range(total): if ok(pos): found = pos break if found is None: raise RuntimeError("Failed to place anchor without adjacency.") anchor_positions.append(found) for pos in anchor_positions: seq[pos] = anchor_idx # 5) 빈 자리를 others로 채우기 j = 0 for i in range(total): if seq[i] is None: seq[i] = others[j] j += 1 # 6) 안전 체크 assert sum(1 for x in seq if x == anchor_idx) == repeats for i in range(1, total): assert not (seq[i] == anchor_idx and seq[i-1] == anchor_idx), "Adjacent anchors found." return seq # def _start_and_load_first(): # total = TOTAL_PER_PARTICIPANT # order = _build_order_with_anchor( # total=total, # anchor_idx=ANCHOR_IDX, # repeats=ANCHOR_REPEATS, # pool_size=len(V), # min_gap=1 # 인접 금지 # ) # first_idx = order[0] # v0 = V[first_idx] # url0 = v0["url"] # action0 = _extract_action(v0) # vid0 = _get_video_id(v0) # ✅ 여기서 원본 id # return ( # gr.update(visible=False), # page_intro off # gr.update(visible=True), # page_eval on # url0, # video # action0, # action_tb (표시용) # 5.0, # score 초기값 # gr.update(visible=False, value=""), # 0, # done_state # _progress_html(0, TOTAL_PER_PARTICIPANT), # order, # order_state # 1, # ptr_state # vid0 # ✅ cur_video_id # ) def _start_and_load_first(): total = TOTAL_PER_PARTICIPANT order = _build_order_least_first_with_anchor( total=total, anchor_idx=ANCHOR_IDX, repeats=ANCHOR_REPEATS, min_gap=MIN_GAP ) first_idx = order[0] v0 = V[first_idx] return ( gr.update(visible=False), gr.update(visible=True), v0["url"], _extract_action(v0), 5.0, gr.update(visible=False, value=""), 0, _progress_html(0, TOTAL_PER_PARTICIPANT), order, 1, _get_video_id(v0) # cur_video_id ) start_btn.click( _start_and_load_first, inputs=[], outputs=[page_intro, page_eval, video, action_tb, score, status, done_state, progress, order_state, ptr_state, cur_video_id] # ✅ ) # -------- Save & Next (노트 없음) -------- def save_and_next(participant_id, action_name, score_val, done_cnt, order, ptr): if not participant_id or not participant_id.strip(): # PID 없으면 기존 화면 유지 return ( gr.update(visible=True, value="❗ Please enter your Participant ID."), gr.update(), gr.update(), # video, action_tb 변경 없음 done_cnt, _progress_html(done_cnt, TOTAL_PER_PARTICIPANT), 5.0, ptr, video_id ) status_msg = push(participant_id, action_name, score_val, "") new_done = int(done_cnt) + 1 # 종료 조건: 목표 개수 달성 or 순서 소진 if new_done >= TOTAL_PER_PARTICIPANT or ptr >= len(order): return ( status_msg, # status None, # video 비우기 "", # action_tb 비우기 TOTAL_PER_PARTICIPANT, # done_state 최종 _progress_html(TOTAL_PER_PARTICIPANT, TOTAL_PER_PARTICIPANT), 5.0, # score 리셋 ptr, video_id ) # 다음 영상 로드 next_idx = order[ptr] v = V[next_idx] return ( status_msg, v["url"], _extract_action(v), new_done, _progress_html(new_done, TOTAL_PER_PARTICIPANT), 5.0, ptr + 1, _get_video_id(v) # ✅ 다음 cur_video_id ) # save_next.click( # save_and_next, # inputs=[pid, action_tb, score, done_state, order_state, ptr_state], # outputs=[status, video, action_tb, done_state, progress, score, ptr_state] # ) save_next.click( save_and_next, # ✅ cur_video_id를 두 번째 인자로 넘김 inputs=[pid, cur_video_id, score, done_state, order_state, ptr_state], # ✅ 마지막에 cur_video_id를 outputs로 받음(상태 갱신) outputs=[status, video, action_tb, done_state, progress, score, ptr_state, cur_video_id] ) if __name__ == "__main__": demo.launch()