Major update of code. Adding new data with our generations

.gitignore

@@ -1,3 +1,5 @@
 benchmark_images_generations/
 code/
 results/
+backup/
+__pycache__/

app.py

@@ -1,396 +1,371 @@
 import gradio as gr
+import pandas as pd
 import os
 import random
-import csv
-from pathlib import Path
-from datetime import datetime, timedelta
-import tempfile
-from huggingface_hub import HfApi, hf_hub_download, login
-from huggingface_hub.utils import RepositoryNotFoundError, EntryNotFoundError
+from datetime import datetime
 from apscheduler.schedulers.background import BackgroundScheduler
-import atexit
-import threading
-import time
-import shutil
-
-# --- Configuration ---
-DATASET_REPO_ID = os.getenv("DATASET_REPO_ID", "matsant01/user-study-collected-preferences")
-HF_TOKEN = os.getenv("HF_TOKEN")
-RESULTS_FILENAME_IN_REPO = "preferences.csv"
-TEMP_DIR = tempfile.mkdtemp()
-LOCAL_RESULTS_FILE = Path(TEMP_DIR) / RESULTS_FILENAME_IN_REPO
-UPLOAD_INTERVAL_HOURS = 0.1
-
-DATA_DIR = Path("data")
-IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".webp"]
-
-# --- Global State for Upload Logic ---
-hf_api = None
-scheduler = BackgroundScheduler(daemon=True)
-upload_lock = threading.Lock()
-new_preferences_recorded_since_last_upload = threading.Event()
-
-# --- Hugging Face Hub Login & Initialization ---
-def initialize_hub_and_results():
-    global hf_api
-    if HF_TOKEN:
-        print("Logging into Hugging Face Hub...")
+from PIL import Image
+
+import config
+import utils
+
+# --- Global Variables & Initial Setup ---
+# Attempt to log in to Hugging Face Hub at startup
+utils.login_hugging_face()
+
+# Load preferences: Try from Hub, then local, then empty
+preferences_df = utils.load_preferences_from_hf_hub(config.HF_DATASET_REPO_ID, config.RESULTS_CSV_FILE)
+if preferences_df is None:
+    if os.path.exists(config.RESULTS_CSV_FILE):
+        print(f"Loading preferences from local file: {config.RESULTS_CSV_FILE}")
         try:
-            login(token=HF_TOKEN)
-            hf_api = HfApi()
-            print(f"Attempting initial download of {RESULTS_FILENAME_IN_REPO} from {DATASET_REPO_ID}")
-            hf_hub_download(
-                repo_id=DATASET_REPO_ID,
-                filename=RESULTS_FILENAME_IN_REPO,
-                repo_type="dataset",
-                token=HF_TOKEN,
-                local_dir=TEMP_DIR,
-                local_dir_use_symlinks=False
-            )
-            print(f"Successfully downloaded existing {RESULTS_FILENAME_IN_REPO} to {LOCAL_RESULTS_FILE}")
-        except EntryNotFoundError:
-            print(f"{RESULTS_FILENAME_IN_REPO} not found in repo. Will create locally.")
-        except RepositoryNotFoundError:
-            print(f"Error: Dataset repository {DATASET_REPO_ID} not found or token lacks permissions.")
-            print("Results saving will be disabled.")
-            hf_api = None
+            preferences_df = pd.read_csv(config.RESULTS_CSV_FILE)
+        except pd.errors.EmptyDataError:
+            print(f"Local preferences file {config.RESULTS_CSV_FILE} is empty. Starting fresh.")
+            preferences_df = pd.DataFrame(columns=config.CSV_HEADERS)
         except Exception as e:
-            print(f"Error during initial download/login: {e}")
-            print("Proceeding without initial download. File will be created locally.")
+            print(f"Error loading local {config.RESULTS_CSV_FILE}: {e}. Starting fresh.")
+            preferences_df = pd.DataFrame(columns=config.CSV_HEADERS)
     else:
-        print("Warning: HF_TOKEN secret not found. Results will not be saved to the Hub.")
-        hf_api = None
-
-# --- Data Loading ---
-
-def find_image(folder_path: Path, base_name: str) -> Path | None:
-    for ext in IMAGE_EXTENSIONS:
-        file_path = folder_path / f"{base_name}{ext}"
-        if file_path.exists():
-            return file_path
-    return None
-
-def get_sample_ids() -> list[str]:
-    sample_ids = []
-    if DATA_DIR.is_dir():
-        for item in DATA_DIR.iterdir():
-            if item.is_dir():
-                prompt_file = item / "prompt.txt"
-                input_bg = find_image(item, "input_bg")
-                input_fg = find_image(item, "input_fg")
-                output_baseline = find_image(item, "baseline")
-                output_tficon = find_image(item, "tf-icon")
-                if prompt_file.exists() and input_bg and input_fg and output_baseline and output_tficon:
-                    sample_ids.append(item.name)
-    return sample_ids
-
-def load_sample_data(sample_id: str) -> dict | None:
-    sample_path = DATA_DIR / sample_id
-    if not sample_path.is_dir():
-        return None
-
-    prompt_file = sample_path / "prompt.txt"
-    input_bg_path = find_image(sample_path, "input_bg")
-    input_fg_path = find_image(sample_path, "input_fg")
-    output_baseline_path = find_image(sample_path, "baseline")
-    output_tficon_path = find_image(sample_path, "tf-icon")
-
-    if not all([prompt_file.exists(), input_bg_path, input_fg_path, output_baseline_path, output_tficon_path]):
-        print(f"Warning: Missing files in sample {sample_id}")
-        return None
-
-    try:
-        prompt = prompt_file.read_text().strip()
-    except Exception as e:
-        print(f"Error reading prompt for {sample_id}: {e}")
-        return None
-
-    return {
-        "id": sample_id,
-        "prompt": prompt,
-        "input_bg": str(input_bg_path),
-        "input_fg": str(input_fg_path),
-        "output_baseline": str(output_baseline_path),
-        "output_tficon": str(output_tficon_path),
-    }
-
-# --- State and UI Logic ---
-
-INITIAL_SAMPLE_IDS = get_sample_ids()
-
-def get_next_sample(available_ids: list[str]) -> tuple[dict | None, list[str]]:
-    if not available_ids:
-        return None, []
-    chosen_id = random.choice(available_ids)
-    remaining_ids = [id for id in available_ids if id != chosen_id]
-    sample_data = load_sample_data(chosen_id)
-    return sample_data, remaining_ids
-
-def display_new_sample(state: dict, available_ids: list[str]):
-    sample_data, remaining_ids = get_next_sample(available_ids)
-
-    if not sample_data:
-        return {
-            prompt_display: gr.update(value="**Prompt:** No more samples available. Thank you!"),
-            input_bg_display: gr.update(value=None, visible=False),
-            input_fg_display: gr.update(value=None, visible=False),
-            output_a_display: gr.update(value=None, visible=False),
-            output_b_display: gr.update(value=None, visible=False),
-            choice_button_a: gr.update(visible=False),
-            choice_button_b: gr.update(visible=False),
-            next_button: gr.update(visible=False),
-            status_display: gr.update(value="**Status:** Completed!"),
-            app_state: state,
-            available_samples_state: remaining_ids
-        }
-
-    outputs = [
-        {"model_name": "baseline", "path": sample_data["output_baseline"]},
-        {"model_name": "tf-icon", "path": sample_data["output_tficon"]},
-    ]
-    random.shuffle(outputs)
-    output_a = outputs[0]
-    output_b = outputs[1]
-
-    state = {
-        "current_sample_id": sample_data["id"],
-        "output_a_model_name": output_a["model_name"],
-        "output_b_model_name": output_b["model_name"],
-    }
-
-    return {
-        prompt_display: gr.update(value=f"**Prompt:** {sample_data['prompt']}"),
-        input_bg_display: gr.update(value=sample_data["input_bg"], visible=True),
-        input_fg_display: gr.update(value=sample_data["input_fg"], visible=True),
-        output_a_display: gr.update(value=output_a["path"], visible=True),
-        output_b_display: gr.update(value=output_b["path"], visible=True),
-        choice_button_a: gr.update(visible=True, interactive=True),
-        choice_button_b: gr.update(visible=True, interactive=True),
-        next_button: gr.update(visible=False),
-        status_display: gr.update(value="**Status:** Please choose the image you prefer."),
-        app_state: state,
-        available_samples_state: remaining_ids
-    }
-
-def record_preference(choice: str, state: dict, request: gr.Request):
-    if not request:
-        print("Error: Request object is None. Cannot get session ID.")
-        session_id = "unknown_session"
+        print("No existing preferences found on Hub or locally. Starting with an empty table.")
+        preferences_df = pd.DataFrame(columns=config.CSV_HEADERS)
+
+# Scan for available data
+ALL_SAMPLES_BY_DOMAIN = utils.scan_data_directory(config.DATA_FOLDER)
+if not ALL_SAMPLES_BY_DOMAIN:
+    print(f"CRITICAL: No data found in {config.DATA_FOLDER}. The app might not function correctly.")
+    # Potentially raise an error or display a message in the UI if no data
+
+# --- Scheduler for Periodic Uploads ---
+def scheduled_upload_job():
+    global preferences_df
+    print(f"Running scheduled job: Saving and uploading preferences at {datetime.now()}")
+    if preferences_df is not None and not preferences_df.empty:
+        utils.save_preferences_to_hf_hub(preferences_df, config.HF_DATASET_REPO_ID, config.RESULTS_CSV_FILE, commit_message="Periodic background update")
     else:
+        print("Scheduled job: Preferences DataFrame is empty. Nothing to upload.")
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(scheduled_upload_job, 'interval', hours=config.PUSH_INTERVAL_HOURS)
+scheduler.start()
+print(f"Scheduler started. Will attempt to upload preferences every {config.PUSH_INTERVAL_HOURS} hour(s).")
+
+
+# --- Core Gradio App Functions ---
+def start_new_session():
+    """Initializes a new user session."""
+    session_id = utils.generate_session_id()
+    sample_queue = utils.prepare_session_samples(ALL_SAMPLES_BY_DOMAIN, config.SAMPLES_PER_DOMAIN)
+    current_sample_index = 0
+    if not sample_queue:
+        no_samples_msg = f"# 😥 No Samples Available!\n\n### Please check the data folder configuration or try again later."
+        return session_id, sample_queue, current_sample_index, no_samples_msg, None, None, None, [], [], True
+    
+    print(f"New session started: {session_id}, with {len(sample_queue)} samples.")
+    domain_prompt_md, bg, fg, s_data, out_imgs, disp_info, end_flag = load_and_display_sample(sample_queue, current_sample_index)
+    return session_id, sample_queue, current_sample_index, domain_prompt_md, bg, fg, s_data, out_imgs, disp_info, end_flag
+
+
+def load_and_display_sample(sample_queue, current_sample_index):
+    """Loads and prepares a single sample for display."""
+    if not sample_queue or current_sample_index >= len(sample_queue):
+        end_session_msg = f"# 🎉 All Rated! 🎉\n\n### All samples for this session have been rated. Thank you!"
+        return end_session_msg, None, None, None, [], [], True # End of session
+
+    domain, sample_id = sample_queue[current_sample_index]
+    sample_data = utils.load_sample_data(domain, sample_id)
+
+    if sample_data is None:
+        print(f"Error loading sample {domain}/{sample_id}. Skipping.")
+        error_msg = f"## ⚠️ Error Loading Sample\n\nCould not load data for {domain}/{sample_id}. Skipping to the next one."
+        return error_msg, None, None, None, [], [], False
+
+    prompt_text = sample_data["prompt"]
+    bg_img_path = sample_data["background_img_path"]
+    fg_img_path = sample_data["foreground_img_path"]
+    
+    # Load input bg/fg images without forcing them to be square
+    # The gr.Image component will handle scaling to the specified height while preserving aspect ratio.
+    bg_image_to_display = Image.open(bg_img_path)
+    fg_image_to_display = Image.open(fg_img_path)
+    
+    output_model_keys = list(sample_data["output_image_paths"].keys())
+    random.shuffle(output_model_keys)
+    
+    displayed_models_info = []
+    output_images_for_display = []
+    
+    # square_size is still used for output option images
+    square_size = (config.IMAGE_DISPLAY_SIZE[0], config.IMAGE_DISPLAY_SIZE[0])
+
+    for model_key in output_model_keys:
+        img_path = sample_data["output_image_paths"][model_key]
         try:
-            session_id = request.client.host
-        except AttributeError:
-            print("Error: request.client is None or has no 'host' attribute.")
-            session_id = "unknown_client"
-
-    if not state or "current_sample_id" not in state:
-        print("Warning: State missing, cannot record preference.")
-        return {
-            choice_button_a: gr.update(interactive=False),
-            choice_button_b: gr.update(interactive=False),
-            next_button: gr.update(visible=True, interactive=True),
-            status_display: gr.update(value="**Status:** Error: Session state lost. Click Next Sample."),
-            app_state: state
-        }
-
-    chosen_model_name = state["output_a_model_name"] if choice == "A" else state["output_b_model_name"]
-    baseline_display = "A" if state["output_a_model_name"] == "baseline" else "B"
-    tficon_display = "B" if state["output_a_model_name"] == "baseline" else "A"
-
-    new_row = {
-        "timestamp": datetime.now().isoformat(),
-        "session_id": session_id,
-        "sample_id": state["current_sample_id"],
-        "baseline_displayed_as": baseline_display,
-        "tficon_displayed_as": tficon_display,
-        "chosen_display": choice,
-        "chosen_model_name": chosen_model_name
-    }
-    header = list(new_row.keys())
-
-    try:
-        with upload_lock:
-            file_exists = LOCAL_RESULTS_FILE.exists()
-            mode = 'a' if file_exists else 'w'
-            with open(LOCAL_RESULTS_FILE, mode, newline='', encoding='utf-8') as f:
-                writer = csv.DictWriter(f, fieldnames=header)
-                if not file_exists or os.path.getsize(LOCAL_RESULTS_FILE) == 0:
-                    writer.writeheader()
-                    print(f"Created or wrote header to {LOCAL_RESULTS_FILE}")
-                writer.writerow(new_row)
-            print(f"Appended preference for {state['current_sample_id']} to local file.")
-            new_preferences_recorded_since_last_upload.set()
-
-    except Exception as e:
-        print(f"Error writing local results file {LOCAL_RESULTS_FILE}: {e}")
-        return {
-            choice_button_a: gr.update(interactive=False),
-            choice_button_b: gr.update(interactive=False),
-            next_button: gr.update(visible=True, interactive=True),
-            status_display: gr.update(value=f"**Status:** Error saving preference locally: {e}. Click Next."),
-            app_state: state
-        }
-
-    return {
-        choice_button_a: gr.update(interactive=False),
-        choice_button_b: gr.update(interactive=False),
-        next_button: gr.update(visible=True, interactive=True),
-        status_display: gr.update(value=f"**Status:** Preference recorded (Chose {choice}). Click Next Sample."),
-        app_state: state
-    }
-
-def upload_preferences_to_hub():
-    print("Periodic upload check triggered.")
-    if not hf_api:
-        print("Upload check skipped: Hugging Face API not available.")
-        return
-
-    if not new_preferences_recorded_since_last_upload.is_set():
-        print("Upload check skipped: No new preferences recorded since last upload.")
-        return
-
-    with upload_lock:
-        if not new_preferences_recorded_since_last_upload.is_set():
-            print("Upload check skipped (race condition avoided): No new preferences.")
-            return
-
-        if not LOCAL_RESULTS_FILE.exists() or os.path.getsize(LOCAL_RESULTS_FILE) == 0:
-            print("Upload check skipped: Local results file is missing or empty.")
-            new_preferences_recorded_since_last_upload.clear()
-            return
-
-        try:
-            print(f"Attempting to upload {LOCAL_RESULTS_FILE} to {DATASET_REPO_ID}/{RESULTS_FILENAME_IN_REPO}")
-            start_time = time.time()
-            hf_api.upload_file(
-                path_or_fileobj=str(LOCAL_RESULTS_FILE),
-                path_in_repo=RESULTS_FILENAME_IN_REPO,
-                repo_id=DATASET_REPO_ID,
-                repo_type="dataset",
-                commit_message=f"Periodic upload of preferences - {datetime.now().isoformat()}"
-            )
-            end_time = time.time()
-            print(f"Successfully uploaded preferences. Took {end_time - start_time:.2f} seconds.")
-            new_preferences_recorded_since_last_upload.clear()
+            img = Image.open(img_path).resize(square_size) # Output images remain square
+            output_images_for_display.append(img)
+            displayed_models_info.append((model_key, img_path)) 
+        except FileNotFoundError:
+            print(f"Image not found: {img_path} for model {model_key}. Skipping this option.")
         except Exception as e:
-            print(f"Error uploading results file: {e}")
-
-def handle_choice_a(state: dict, request: gr.Request):
-    return record_preference("A", state, request)
-
-def handle_choice_b(state: dict, request: gr.Request):
-    return record_preference("B", state, request)
-
-with gr.Blocks(title="Image Composition User Study") as demo:
-    gr.Markdown("# Image Composition User Study")
-    gr.Markdown(
-        "> Please look at the input images and the prompt below. "
-        "Then, compare the two output images (Output A and Output B) and click the button below the one you prefer."
+            print(f"Error loading or resizing image {img_path}: {e}. Skipping this option.")
+
+    blank_image = Image.new('RGB', square_size, (200, 200, 200))
+    while len(output_images_for_display) < 4:
+        output_images_for_display.append(blank_image)
+        displayed_models_info.append(("BLANK_SLOT", "N/A"))
+
+    domain_prompt_markdown = f"""
+    <div style="padding:15px 20px 20px 20px;border-left:3px black;background-color:#4B5966;border-radius: 10px;color:black;">
+    
+    ### Domain: {domain}
+
+    </div>
+    <br>
+    <div style="padding:15px 20px 20px 20px;border-left:3px black;background-color:#4B5966;border-radius: 10px;color:black;">
+    
+    ## Prompt        
+        
+    ### _"{prompt_text}"_
+    
+    </div>
+    """
+    
+    return (
+        domain_prompt_markdown,
+        bg_image_to_display, # Pass the PIL image directly
+        fg_image_to_display, # Pass the PIL image directly
+        sample_data,
+        output_images_for_display[:4],
+        displayed_models_info[:4],
+        False
     )
 
-    app_state = gr.State({})
-    available_samples_state = gr.State(INITIAL_SAMPLE_IDS)
-
-    status_display = gr.Markdown("**Status:** Loading first sample...")
+def process_vote(choice_index, session_id, sample_queue, current_sample_index, current_sample_data, displayed_models_info_for_sample):
+    global preferences_df
+    
+    if current_sample_data is None or not displayed_models_info_for_sample or choice_index >= len(displayed_models_info_for_sample):
+        print("Error: Invalid data for processing vote. Skipping.")
+        current_sample_index += 1
+        if current_sample_index >= len(sample_queue):
+            error_end_msg = f"# ⚠️ Error Processing Vote ⚠️\n\n### An issue occurred. The session has ended."
+            return preferences_df, current_sample_index, error_end_msg, None, None, None, [], [], True
+        else:
+            next_prompt_md, next_bg, next_fg, next_s_data, next_out_imgs, next_disp_info, next_hide = load_and_display_sample(sample_queue, current_sample_index)
+            return preferences_df, current_sample_index, next_prompt_md, next_bg, next_fg, next_s_data, next_out_imgs, next_disp_info, next_hide
+
+    domain, sample_id = sample_queue[current_sample_index]
+    preferred_model_key, _ = displayed_models_info_for_sample[choice_index]
+
+    if preferred_model_key == "BLANK_SLOT":
+        print("User clicked on a blank slot. Vote not recorded. Please select a valid image.")
+        _prompt_md, _bg, _fg, _s_data, _out_imgs, _disp_info, _hide = load_and_display_sample(sample_queue, current_sample_index)
+        return preferences_df, current_sample_index, _prompt_md, _bg, _fg, _s_data, _out_imgs, _disp_info, _hide
+
+    print(f"Session {session_id}: Voted for model '{config.MODEL_DISPLAY_NAMES.get(preferred_model_key, preferred_model_key)}' (key: {preferred_model_key}) for sample {domain}/{sample_id}")
+
+    preferences_df = utils.record_preference(
+        df=preferences_df,
+        session_id=session_id,
+        domain=domain,
+        sample_id=sample_id,
+        prompt=current_sample_data["prompt"],
+        bg_path=current_sample_data["background_img_path"],
+        fg_path=current_sample_data["foreground_img_path"],
+        displayed_models_info=displayed_models_info_for_sample,
+        preferred_model_key=preferred_model_key
+    )
+    
+    try:
+        preferences_df.to_csv(config.RESULTS_CSV_FILE, index=False)
+        print(f"Preferences saved locally to {config.RESULTS_CSV_FILE}")
+    except Exception as e:
+        print(f"Error saving preferences locally: {e}")
+
+    current_sample_index += 1
+    if current_sample_index >= len(sample_queue):
+        utils.save_preferences_to_hf_hub(preferences_df, config.HF_DATASET_REPO_ID, config.RESULTS_CSV_FILE, commit_message="Session end update")
+        final_msg = f"# 🎉 Session Complete! 🎉\n\n### All samples have been rated. Thank you for your participation!"
+        return preferences_df, current_sample_index, final_msg, None, None, None, [], [], True
+    
+    next_prompt_md, next_bg, next_fg, next_s_data, next_out_imgs, next_disp_info, next_hide = load_and_display_sample(sample_queue, current_sample_index)
+    return preferences_df, current_sample_index, next_prompt_md, next_bg, next_fg, next_s_data, next_out_imgs, next_disp_info, next_hide
+
+
+# --- Gradio UI Definition ---
+custom_css = """
+.custom-vote-button {
+    background-color: #FFA500 !important; /* Light Orange for normal state */
+    border-color: #FFA500 !important; /* Light Orange for normal state */
+    color: white !important;
+}
+.custom-vote-button:hover {
+    background-color: #FF8C00 !important; /* Dark Orange for hover state */
+    border-color: #FF8C00 !important; /* Dark Orange for hover state */
+    color: white !important;
+}
+"""
+
+with gr.Blocks(title=config.APP_TITLE, theme=gr.themes.Soft(primary_hue=gr.themes.colors.blue), css=custom_css) as demo:
+    session_id_state = gr.State()
+    sample_queue_state = gr.State([])
+    current_sample_index_state = gr.State(0)
+    current_sample_data_state = gr.State() 
+    displayed_models_info_state = gr.State([]) 
+    preferences_df_state = gr.State(value=preferences_df)
+
+    gr.Markdown(f"# {config.APP_TITLE}")
+    gr.Markdown(config.APP_DESCRIPTION)
 
-    gr.Markdown("## Inputs")
-    with gr.Row():
-        prompt_display = gr.Markdown("**Prompt:** Loading...")
     with gr.Row():
-        with gr.Column():
-            gr.Markdown("<div style='text-align: center;'>Input Background</div>")
-            input_bg_display = gr.Image(type="filepath", height=250, width=250, interactive=False, show_label=False)
-        with gr.Column():
-            gr.Markdown("<div style='text-align: center;'>Input Foreground</div>")
-            input_fg_display = gr.Image(type="filepath", height=250, width=250, interactive=False, show_label=False)
+        start_button = gr.Button("Start New Session / Load First Sample", variant="primary")
+    
+    with gr.Row(equal_height=False):
+        with gr.Column(scale=1):
+            domain_prompt_info_display = gr.Markdown(value="### Click 'Start New Session' to begin.")
+        
+        with gr.Column(scale=2):
+            with gr.Row():
+                input_bg_image_display = gr.Image(label="Input Background", type="pil", height=config.IMAGE_DISPLAY_SIZE[0], interactive=False)
+                input_fg_image_display = gr.Image(label="Input Foreground", type="pil", height=config.IMAGE_DISPLAY_SIZE[0], interactive=False)
 
     gr.Markdown("---")
-    gr.Markdown("## Choose your preferred output")
+    gr.Markdown("## Choose your preferred composed image:")
 
+    output_image_displays = []
+    vote_buttons = []
     with gr.Row():
-        with gr.Column():
-            output_a_display = gr.Image(label="Output A", type="filepath", height=400, width=400, interactive=False)
-            choice_button_a = gr.Button("Choose Output A", variant="primary")
-        with gr.Column():
-            output_b_display = gr.Image(label="Output B", type="filepath", height=400, width=400, interactive=False)
-            choice_button_b = gr.Button("Choose Output B", variant="primary")
-
-    next_button = gr.Button("🔁 Next Sample 🔁", visible=False)
-
-    demo.load(
-        fn=display_new_sample,
-        inputs=[app_state, available_samples_state],
+        for i in range(4):
+            with gr.Column():
+                img_display = gr.Image(label=f"Option {i+1}", type="pil", height=config.IMAGE_DISPLAY_SIZE[0], width=config.IMAGE_DISPLAY_SIZE[0], interactive=False)
+                output_image_displays.append(img_display)
+                vote_btn = gr.Button(f"Select Option {i+1}", elem_id=f"vote_btn_{i}", elem_classes=["custom-vote-button"])
+                vote_buttons.append(vote_btn)
+    
+    end_of_session_msg_display = gr.Markdown("", visible=True)
+
+    def handle_start_session():
+        s_id, s_queue, s_idx, domain_prompt_or_end_msg, bg, fg, s_data, out_imgs, disp_info, end = start_new_session()
+        
+        while len(out_imgs) < 4: out_imgs.append(None)
+        while len(disp_info) < 4: disp_info.append(("BLANK_SLOT", "N/A"))
+        
+        updates = {
+            session_id_state: s_id,
+            sample_queue_state: s_queue,
+            current_sample_index_state: s_idx,
+            domain_prompt_info_display: domain_prompt_or_end_msg if not end else "",
+            input_bg_image_display: bg,
+            input_fg_image_display: fg,
+            current_sample_data_state: s_data,
+            displayed_models_info_state: disp_info,
+            end_of_session_msg_display: domain_prompt_or_end_msg if end else ""
+        }
+        for i in range(4):
+            updates[output_image_displays[i]] = out_imgs[i] if i < len(out_imgs) else None
+            num_actual_outputs = 0
+            if s_data and "output_image_paths" in s_data and s_data["output_image_paths"]:
+                 num_actual_outputs = sum(1 for m_key, _ in disp_info if m_key != "BLANK_SLOT" and m_key is not None)
+            updates[vote_buttons[i]] = gr.Button(interactive=not end and i < num_actual_outputs)
+        return updates
+
+    start_button.click(
+        fn=handle_start_session,
+        inputs=[],
         outputs=[
-            prompt_display, input_bg_display, input_fg_display,
-            output_a_display, output_b_display,
-            choice_button_a, choice_button_b, next_button, status_display,
-            app_state, available_samples_state
+            session_id_state, sample_queue_state, current_sample_index_state,
+            domain_prompt_info_display,
+            input_bg_image_display, input_fg_image_display,
+            current_sample_data_state, displayed_models_info_state, end_of_session_msg_display,
+            *output_image_displays, *vote_buttons
         ]
     )
 
-    choice_button_a.click(
-        fn=handle_choice_a,
-        inputs=[app_state],
-        outputs=[choice_button_a, choice_button_b, next_button, status_display, app_state],
-        api_name=False,
-    )
+    def make_vote_fn(choice_idx):
+        def vote_action(s_id, s_queue, s_idx, current_s_data, disp_info_for_sample, prefs_df_val):
+            global preferences_df
+            preferences_df = prefs_df_val
 
-    choice_button_b.click(
-        fn=handle_choice_b,
-        inputs=[app_state],
-        outputs=[choice_button_a, choice_button_b, next_button, status_display, app_state],
-        api_name=False,
-    )
-
-    next_button.click(
-        fn=display_new_sample,
-        inputs=[app_state, available_samples_state],
-        outputs=[
-            prompt_display, input_bg_display, input_fg_display,
-            output_a_display, output_b_display,
-            choice_button_a, choice_button_b, next_button, status_display,
-            app_state, available_samples_state
-        ],
-        api_name=False,
-    )
-
-def cleanup_temp_dir():
-    if Path(TEMP_DIR).exists():
-        print(f"Cleaning up temporary directory: {TEMP_DIR}")
-        shutil.rmtree(TEMP_DIR, ignore_errors=True)
-
-def shutdown_hook():
-    print("Application shutting down. Performing final upload check...")
-    upload_preferences_to_hub()
-    if scheduler.running:
-        print("Shutting down scheduler...")
-        scheduler.shutdown(wait=False)
-    cleanup_temp_dir()
-    print("Shutdown complete.")
-
-atexit.register(shutdown_hook)
+            new_prefs_df, new_s_idx, domain_prompt_or_end_msg, bg, fg, new_s_data, out_imgs, new_disp_info, end = process_vote(
+                choice_idx, s_id, s_queue, s_idx, current_s_data, disp_info_for_sample
+            )
+            
+            while len(out_imgs) < 4: out_imgs.append(None)
+            while len(new_disp_info) < 4: new_disp_info.append(("BLANK_SLOT", "N/A"))
+
+            updates = {
+                preferences_df_state: new_prefs_df,
+                current_sample_index_state: new_s_idx,
+                domain_prompt_info_display: domain_prompt_or_end_msg if not end else "",
+                input_bg_image_display: bg,
+                input_fg_image_display: fg,
+                current_sample_data_state: new_s_data,
+                displayed_models_info_state: new_disp_info,
+                end_of_session_msg_display: domain_prompt_or_end_msg if end else ""
+            }
+            for i in range(4):
+                updates[output_image_displays[i]] = out_imgs[i] if i < len(out_imgs) else None
+                num_actual_outputs = 0
+                if new_s_data and "output_image_paths" in new_s_data and new_s_data["output_image_paths"]:
+                    num_actual_outputs = sum(1 for m_key, _ in new_disp_info if m_key != "BLANK_SLOT" and m_key is not None)
+                updates[vote_buttons[i]] = gr.Button(interactive=not end and i < num_actual_outputs)
+            return updates
+        return vote_action
+
+    for i, btn in enumerate(vote_buttons):
+        btn.click(
+            fn=make_vote_fn(i),
+            inputs=[
+                session_id_state, sample_queue_state, current_sample_index_state,
+                current_sample_data_state, displayed_models_info_state, preferences_df_state
+            ],
+            outputs=[
+                preferences_df_state, current_sample_index_state,
+                domain_prompt_info_display,
+                input_bg_image_display, input_fg_image_display,
+                current_sample_data_state, displayed_models_info_state, end_of_session_msg_display,
+                *output_image_displays, *vote_buttons
+            ]
+        )
+
+    gr.Markdown(config.FOOTER_MESSAGE)
 
 if __name__ == "__main__":
-    initialize_hub_and_results()
-
-    if not INITIAL_SAMPLE_IDS:
-        print("Error: No valid samples found in the 'data' directory.")
-        print("Please ensure the 'data' directory exists and contains subdirectories")
-        print("named like 'sample_id', each with 'prompt.txt', 'input_bg.*',")
-        print("'input_fg.*', 'baseline.*', and 'tf-icon.*' files.")
-    elif not DATASET_REPO_ID:
-        print("Error: DATASET_REPO_ID environment variable is not set or is set to the default placeholder.")
-        print("Please set the DATASET_REPO_ID environment variable or update the script.")
-    elif hf_api:
-        print(f"Starting periodic upload scheduler (every {UPLOAD_INTERVAL_HOURS} hours)...")
-        scheduler.add_job(upload_preferences_to_hub, 'interval', hours=UPLOAD_INTERVAL_HOURS)
-        scheduler.start()
-        print(f"Found {len(INITIAL_SAMPLE_IDS)} samples.")
-        print(f"Configured to save results periodically to Hugging Face Dataset: {DATASET_REPO_ID}")
-        print("Starting Gradio app...")
-        demo.launch(server_name="0.0.0.0")
-    else:
-        print("Warning: Running without Hugging Face Hub integration (HF_TOKEN or DATASET_REPO_ID missing/invalid).")
-        print(f"Found {len(INITIAL_SAMPLE_IDS)} samples.")
-        print("Starting Gradio app...")
-        demo.launch(server_name="0.0.0.0")
+    if not os.path.exists(config.DATA_FOLDER):
+        print(f"Creating dummy data folder: {config.DATA_FOLDER}")
+        os.makedirs(config.DATA_FOLDER, exist_ok=True)
+        
+        dummy_domains = ["Real-Cartoon", "Real-Painting"]
+        dummy_model_keys = list(config.MODEL_OUTPUT_IMAGE_NAMES.keys())
+
+        for domain in dummy_domains:
+            domain_path = os.path.join(config.DATA_FOLDER, domain)
+            os.makedirs(domain_path, exist_ok=True)
+            for i in range(config.SAMPLES_PER_DOMAIN + 2):
+                sample_id = f"sample_{i:03d}"
+                sample_path = os.path.join(domain_path, sample_id)
+                os.makedirs(sample_path, exist_ok=True)
+
+                with open(os.path.join(sample_path, config.PROMPT_FILE_NAME), "w") as f:
+                    f.write(f"This is a dummy prompt for {domain} sample {sample_id}.")
+                
+                colors = [(255,0,0), (0,255,0), (0,0,255), (255,255,0), (0,255,255)]
+                try:
+                    img_bg = Image.new('RGB', config.IMAGE_DISPLAY_SIZE, color='gray')
+                    img_bg.save(os.path.join(sample_path, config.BACKGROUND_IMAGE_NAME))
+                    
+                    img_fg = Image.new('RGB', config.IMAGE_DISPLAY_SIZE, color='lightgray')
+                    img_fg.save(os.path.join(sample_path, config.FOREGROUND_IMAGE_NAME))
+
+                    for idx, model_key in enumerate(dummy_model_keys):
+                        model_img_name = config.MODEL_OUTPUT_IMAGE_NAMES[model_key]
+                        img_model = Image.new('RGB', config.IMAGE_DISPLAY_SIZE, color=colors[idx % len(colors)])
+                        img_model.save(os.path.join(sample_path, model_img_name))
+                except Exception as e:
+                    print(f"Error creating dummy image: {e}")
+        print("Dummy data creation complete.")
+        ALL_SAMPLES_BY_DOMAIN = utils.scan_data_directory(config.DATA_FOLDER)
+
+    demo.launch()
+
+import atexit
+atexit.register(lambda: scheduler.shutdown() if scheduler.running else None)

clean_preferences.py

@@ -0,0 +1,104 @@
+import os
+import datasets
+import pandas as pd
+from datetime import datetime
+
+from config import BACKUP_FOLDER, HF_DATASET_REPO_ID, HF_TOKEN, RESULTS_CSV_FILE, CSV_HEADERS
+
+def main():
+    """
+    Gets the dataset from HF Hub where preferences are being collected,
+    save it locally to a backup folder with a timestamp.
+    Then creates an empty dataset with the same structure and saves it to the HF Hub.
+    """
+    print(f"Attempting to load dataset '{HF_DATASET_REPO_ID}' from Hugging Face Hub (file: {RESULTS_CSV_FILE})...")
+    try:
+        # 1. Get the dataset from HF Hub
+        # Ensure the token has write permissions for pushing later.
+        dataset = datasets.load_dataset(HF_DATASET_REPO_ID, data_files=RESULTS_CSV_FILE, token=HF_TOKEN, split='train')
+        print(f"Successfully loaded dataset. It has {len(dataset)} entries.")
+        dataset_df = dataset.to_pandas()
+    except Exception as e:
+        print(f"Error loading dataset from Hugging Face Hub: {e}")
+        print("This could be due to the dataset/file not existing, or token issues.")
+        print("Attempting to proceed by creating an empty structure for backup and remote reset.")
+        # If loading fails, we might still want to try to clear the remote
+        # or at least create an empty local backup structure.
+        dataset_df = pd.DataFrame(columns=CSV_HEADERS) # Use predefined headers
+
+    # 2. Save it locally to a backup folder with a timestamp
+    if not os.path.exists(BACKUP_FOLDER):
+        os.makedirs(BACKUP_FOLDER)
+        print(f"Created backup folder: {BACKUP_FOLDER}")
+
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    backup_filename = f"preferences_backup_{timestamp}.csv"
+    backup_filepath = os.path.join(BACKUP_FOLDER, backup_filename)
+
+    try:
+        dataset_df.to_csv(backup_filepath, index=False)
+        print(f"Successfully backed up current preferences (or empty structure) to: {backup_filepath}")
+    except Exception as e:
+        print(f"Error saving backup to {backup_filepath}: {e}")
+        # Decide if to return or continue to try clearing remote
+        # For now, let's continue to try clearing remote if backup fails
+
+    # 3. Create an empty dataset with the same structure (using config.CSV_HEADERS)
+    print(f"Creating an empty dataset structure using predefined CSV_HEADERS: {CSV_HEADERS}")
+    empty_df = pd.DataFrame(columns=CSV_HEADERS)
+    empty_dataset = datasets.Dataset.from_pandas(empty_df)
+
+    # 4. Save the empty dataset to the HF Hub
+    print(f"Attempting to push the empty dataset to '{HF_DATASET_REPO_ID}' (file: {RESULTS_CSV_FILE}) on Hugging Face Hub...")
+    try:
+        # To push a specific CSV file and overwrite it, we can push a dictionary
+        # where the key is the name of the file in the repo (without .csv extension if that's how load_dataset names splits)
+        # or more robustly, save to a local temp CSV and use that path in push_to_hub.
+        
+        # Create a DatasetDict. The key 'train' is a common default split name.
+        # If your dataset on the Hub uses a different split name for this CSV, adjust accordingly.
+        # Or, if RESULTS_CSV_FILE is the exact filename on the hub, that's what we want to replace.
+        dataset_dict_to_push = datasets.DatasetDict({"train": empty_dataset}) 
+
+        # The push_to_hub for a DatasetDict will typically create Parquet files by default.
+        # To ensure it's a CSV, we might need to save it locally first and then push that file.
+        # However, let's try pushing the DatasetDict directly first, as it might handle CSVs
+        # if the original dataset was loaded as such.
+        # For more direct control over pushing a CSV file:
+        temp_empty_csv_path = "_temp_empty_prefs.csv"
+        empty_df.to_csv(temp_empty_csv_path, index=False)
+        
+        # The `push_to_hub` method on a Dataset object itself can be used.
+        # To ensure it overwrites the correct file, it's often best to structure it as a DatasetDict
+        # or manage file uploads more directly if the library offers it for specific file types.
+
+        # Let's use a method that's common for replacing a dataset with a new version from a local file.
+        # We'll upload our temporary empty CSV.
+        # This requires the `huggingface_hub` library to be installed and logged in.
+        from huggingface_hub import HfApi
+        api = HfApi(token=os.getenv("HF_HUB_TOKEN", HF_TOKEN))
+
+        api.upload_file(
+            path_or_fileobj=temp_empty_csv_path,
+            path_in_repo=RESULTS_CSV_FILE, # This should be the path to the CSV file in the repo
+            repo_id=HF_DATASET_REPO_ID,
+            repo_type="dataset",
+            commit_message=f"Reset {RESULTS_CSV_FILE} to empty by script"
+        )
+
+        if os.path.exists(temp_empty_csv_path):
+            os.remove(temp_empty_csv_path)
+
+        print(f"Successfully pushed empty dataset to replace {RESULTS_CSV_FILE} in Hugging Face Hub: {HF_DATASET_REPO_ID}")
+        print("The remote dataset CSV should now be empty but retain its structure based on CSV_HEADERS.")
+        print(f"IMPORTANT: The old data (if any) is backed up at {backup_filepath}")
+
+    except Exception as e:
+        print(f"Error pushing empty dataset to Hugging Face Hub: {e}")
+        if os.path.exists(temp_empty_csv_path):
+            os.remove(temp_empty_csv_path)
+        print("The remote dataset might not have been cleared. Please check the Hugging Face Hub.")
+
+
+if __name__ == "__main__":
+    main()

config.py

@@ -0,0 +1,64 @@
+# Configuration for the Gradio User Study App
+
+# --- File and Folder Names ---
+DATA_FOLDER = "data"  # Main folder containing domain subfolders
+BACKGROUND_IMAGE_NAME = "input_bg.jpg"  # Standard name for background input
+FOREGROUND_IMAGE_NAME = "input_fg.jpg"  # Standard name for foreground input
+PROMPT_FILE_NAME = "prompt.txt"  # Standard name for the prompt file
+# Names for the output images from different models.
+# These should be actual filenames present in each sample's folder.
+MODEL_OUTPUT_IMAGE_NAMES = {
+    "baseline": "cp_bg_fg.jpg",
+    "kv-edit": "kvedit.jpg",
+    "tf-icon": "tf-icon.png",
+    "dit-editor": "alphanoise0.05_timesteps50_QTrue_KTrue_VTrue_taua0.4_taub0.8_guidance3.0.png",
+}
+# Names to display for each model in the UI (can be different from filenames)
+MODEL_DISPLAY_NAMES = {
+    "baseline": "Model A",
+    "kv-edit": "Model B",
+    "tf-icon": "Model C",
+    "dit-editor": "Model D",
+}
+
+# --- Data Collection ---
+RESULTS_CSV_FILE = "user_preferences.csv"
+CSV_HEADERS = [
+    "session_id",
+    "timestamp",
+    "domain",
+    "sample_id",
+    "prompt",
+    "input_background",
+    "input_foreground",
+    "displayed_order_model_1", # To store which model was shown in 1st position
+    "displayed_order_model_2", # To store which model was shown in 2nd position
+    "displayed_order_model_3", # To store which model was shown in 3rd position
+    "displayed_order_model_4", # To store which model was shown in 4th position
+    "preferred_model_key",     # The key of the preferred model (e.g., "model_a")
+    "preferred_model_filename" # The filename of the preferred image
+]
+SAMPLES_PER_DOMAIN = 3 # Number of samples to show from each domain per user session
+
+# --- Hugging Face Hub ---
+HF_DATASET_REPO_ID = "matsant01/dit-editor-collected-preferences"  # Replace with your actual repo ID
+HF_TOKEN = None  # Set this if your dataset is private, or use HF_HUB_TOKEN env var
+PUSH_INTERVAL_HOURS = 1 # Interval in hours to push results to the Hub
+
+# --- UI Configuration ---
+IMAGE_DISPLAY_SIZE = (300, 300) # (width, height) for displaying images
+APP_TITLE = "Image Composition User Study"
+APP_DESCRIPTION = """
+Please look at the input foreground and background images, and the text prompt used for generation, then choose the composed image that you prefer the most.
+You consider:
+* 📸 **subject consistency**: does the subject resemble the one in the foreground image? Or is it just a similar object/animal?
+* 🖼️ **background preservation**: is the background image correctly preserved?
+* 🎨 **style blending**: is the subject style correctly adapted to the one of the background?
+"""
+FOOTER_MESSAGE = "Thank you for participating!"
+
+# --- Other ---
+SESSION_ID_LENGTH = 16 # Length of the randomly generated session ID
+
+# --- Paths ---
+BACKUP_FOLDER = "backup"

data/Real-Cartoon/sample_0/prompt.txt

@@ -0,0 +1 @@
+a cartoon animation of a hamburger, a croissant, a piece of bread and a cup of coffee

data/Real-Cartoon/sample_1/prompt.txt

@@ -0,0 +1 @@
+a cartoon animation of a muffin, a croissant, a piece of bread and a cup of coffee

data/Real-Cartoon/sample_10/prompt.txt

@@ -0,0 +1 @@
+a cartoon animation of a shopping mall in the distance

data/Real-Cartoon/sample_11/prompt.txt

@@ -0,0 +1 @@
+a cartoon animation of a panda in the forest

data/Real-Cartoon/sample_12/prompt.txt

@@ -0,0 +1 @@
+a cartoon animation of buildings in the distance

data/Real-Cartoon/sample_13/prompt.txt

@@ -0,0 +1 @@
+a cartoon animation of a fox in the forest