Upload folder using huggingface_hub

server/ui.py

@@ -143,7 +143,7 @@ def render_home_page() -> str:
       color: var(--muted);
     }
 
-    input {
+    input, select {
       width: 100%;
       border: 1px solid rgba(18, 88, 63, 0.22);
       border-radius: 10px;
@@ -154,12 +154,82 @@ def render_home_page() -> str:
       transition: border-color 180ms ease, box-shadow 180ms ease;
     }
 
-    input:focus {
+    input:focus, select:focus {
       outline: none;
       border-color: var(--accent);
       box-shadow: 0 0 0 4px rgba(15, 139, 99, 0.14);
     }
 
+    .task-picker {
+      display: grid;
+      gap: 8px;
+      padding: 10px;
+      border-radius: 12px;
+      border: 1px solid rgba(15, 139, 99, 0.22);
+      background: rgba(255, 255, 255, 0.85);
+    }
+
+    .task-picker-row {
+      display: grid;
+      grid-template-columns: 1fr auto;
+      gap: 8px;
+    }
+
+    .btn-add-task {
+      background: #1c6f50;
+      color: #fff;
+      padding: 0 14px;
+    }
+
+    .task-chips {
+      display: flex;
+      gap: 8px;
+      flex-wrap: wrap;
+      min-height: 32px;
+      align-items: center;
+    }
+
+    .task-chip {
+      display: inline-flex;
+      align-items: center;
+      gap: 8px;
+      border: 1px solid rgba(20, 67, 50, 0.32);
+      border-radius: 999px;
+      background: #fff8dc;
+      color: #2a362f;
+      padding: 6px 10px;
+      font: 600 12px/1 var(--display);
+    }
+
+    .task-chip button {
+      border: 0;
+      background: transparent;
+      color: #4a2a12;
+      padding: 0;
+      border-radius: 0;
+      min-width: unset;
+      font: 700 12px/1 var(--display);
+      line-height: 1;
+      cursor: pointer;
+    }
+
+    .task-chip button:hover {
+      transform: none;
+      filter: brightness(0.92);
+    }
+
+    .task-empty {
+      font: 500 12px/1.2 var(--mono);
+      color: #52695f;
+    }
+
+    .field-note {
+      margin: 0;
+      color: #4e655b;
+      font-size: 12px;
+      line-height: 1.4;
+    }
+
     .actions {
       margin-top: 6px;
       display: flex;
@@ -318,7 +388,7 @@ def render_home_page() -> str:
     <section class="hero">
       <span class="eyebrow"><span class="dot"></span>FlakySleuth Space</span>
       <h1>Run Inference From The Browser</h1>
-      <p>Launch <code>inference.py</code>, monitor stdout live, and inspect episode summaries without leaving this Space. The API endpoints for OpenEnv stay available at the same time.</p>
+      <p>This evaluator console runs benchmark episodes for <strong>classification</strong>, <strong>root-cause identification</strong>, and <strong>fix proposal</strong>. Use it to review logs, score trends, and reproducible run settings while judging submission quality.</p>
     </section>
 
     <section class="panel-grid">
@@ -341,8 +411,18 @@ def render_home_page() -> str:
           </div>
 
           <div class="field span-2">
-            <label for="task_types">Task Types (comma-separated)</label>
-            <input id="task_types" name="task_types" value="classify,root_cause,fix_proposal" />
+            <label for="task-type-select">Task Types</label>
+            <div class="task-picker">
+              <div id="task-chips" class="task-chips"></div>
+              <div class="task-picker-row">
+                <select id="task-type-select">
+                  <option value="">Choose task type</option>
+                </select>
+                <button id="btn-add-task" class="btn-add-task" type="button">Add</button>
+              </div>
+              <input id="task_types" name="task_types" type="hidden" value="classify,root_cause,fix_proposal" />
+            </div>
+            <p class="field-note">Add from dropdown, remove with <code>x</code> on each chip.</p>
           </div>
 
           <div class="field span-2">
@@ -410,6 +490,106 @@ def render_home_page() -> str:
     const logEl = document.getElementById("log-output");
     const logCountEl = document.getElementById("log-count");
     const summaryEl = document.getElementById("summary-line");
+    const taskInput = document.getElementById("task_types");
+    const taskChipsEl = document.getElementById("task-chips");
+    const taskSelectEl = document.getElementById("task-type-select");
+    const taskAddButton = document.getElementById("btn-add-task");
+
+    const TASK_TYPE_ORDER = ["classify", "root_cause", "fix_proposal"];
+    const TASK_TYPE_LABELS = {
+      classify: "Classify",
+      root_cause: "Root Cause",
+      fix_proposal: "Fix Proposal",
+    };
+
+    function parseTaskTypes(raw) {
+      const tokens = String(raw || "")
+        .split(",")
+        .map((token) => token.trim())
+        .filter(Boolean);
+      const unique = [];
+      for (const token of tokens) {
+        if (TASK_TYPE_ORDER.includes(token) && !unique.includes(token)) {
+          unique.push(token);
+        }
+      }
+      return unique;
+    }
+
+    let selectedTaskTypes = parseTaskTypes(taskInput.value);
+    if (!selectedTaskTypes.length) {
+      selectedTaskTypes = [...TASK_TYPE_ORDER];
+    }
+
+    function renderTaskSelect() {
+      taskSelectEl.innerHTML = "";
+
+      const placeholder = document.createElement("option");
+      placeholder.value = "";
+      placeholder.textContent = "Choose task type";
+      placeholder.selected = true;
+      taskSelectEl.appendChild(placeholder);
+
+      for (const taskType of TASK_TYPE_ORDER) {
+        if (selectedTaskTypes.includes(taskType)) continue;
+        const option = document.createElement("option");
+        option.value = taskType;
+        option.textContent = TASK_TYPE_LABELS[taskType];
+        taskSelectEl.appendChild(option);
+      }
+
+      const hasChoices = taskSelectEl.options.length > 1;
+      taskSelectEl.disabled = !hasChoices;
+      taskAddButton.disabled = !hasChoices;
+    }
+
+    function renderTaskChips() {
+      taskChipsEl.innerHTML = "";
+
+      if (!selectedTaskTypes.length) {
+        const hint = document.createElement("span");
+        hint.className = "task-empty";
+        hint.textContent = "No task selected yet.";
+        taskChipsEl.appendChild(hint);
+        return;
+      }
+
+      for (const taskType of selectedTaskTypes) {
+        const chip = document.createElement("span");
+        chip.className = "task-chip";
+
+        const chipText = document.createElement("span");
+        chipText.textContent = TASK_TYPE_LABELS[taskType] || taskType;
+
+        const chipRemove = document.createElement("button");
+        chipRemove.type = "button";
+        chipRemove.textContent = "x";
+        chipRemove.setAttribute("aria-label", `Remove ${chipText.textContent}`);
+        chipRemove.addEventListener("click", () => {
+          selectedTaskTypes = selectedTaskTypes.filter((value) => value !== taskType);
+          syncTaskTypes();
+        });
+
+        chip.appendChild(chipText);
+        chip.appendChild(chipRemove);
+        taskChipsEl.appendChild(chip);
+      }
+    }
+
+    function syncTaskTypes() {
+      taskInput.value = selectedTaskTypes.join(",");
+      renderTaskChips();
+      renderTaskSelect();
+    }
+
+    function addSelectedTaskType() {
+      const selected = taskSelectEl.value.trim();
+      if (!selected) return;
+      if (!selectedTaskTypes.includes(selected)) {
+        selectedTaskTypes.push(selected);
+      }
+      syncTaskTypes();
+    }
 
     function readFormPayload() {
       return {
@@ -480,6 +660,10 @@ def render_home_page() -> str:
     async function startRun() {
       runButton.disabled = true;
       try {
+        if (!selectedTaskTypes.length) {
+          alert("Please add at least one task type.");
+          return;
+        }
         const response = await fetch("/web/inference/start", {
           method: "POST",
           headers: { "Content-Type": "application/json" },
@@ -497,6 +681,10 @@ def render_home_page() -> str:
         alert("Could not start inference. Check logs and try again.");
       } finally {
         form.api_key.value = "";
+        const isActive = ["running", "starting"].includes((statusText.textContent || "").toLowerCase());
+        if (!isActive) {
+          runButton.disabled = false;
+        }
       }
     }
 
@@ -512,6 +700,15 @@ def render_home_page() -> str:
 
     runButton.addEventListener("click", startRun);
     stopButton.addEventListener("click", stopRun);
+    taskAddButton.addEventListener("click", addSelectedTaskType);
+    taskSelectEl.addEventListener("keydown", (event) => {
+      if (event.key === "Enter") {
+        event.preventDefault();
+        addSelectedTaskType();
+      }
+    });
+
+    syncTaskTypes();
 
     fetchStatus();
     window.setInterval(fetchStatus, 2200);

validate-submission.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+#
+# validate-submission.sh — OpenEnv Submission Validator
+#
+# Checks that your HF Space is live, Docker image builds, and openenv validate passes.
+#
+# Prerequisites:
+#   - Docker:       https://docs.docker.com/get-docker/
+#   - openenv-core: pip install openenv-core
+#   - curl (usually pre-installed)
+#
+# Run:
+#   curl -fsSL https://raw.githubusercontent.com/<owner>/<repo>/main/scripts/validate-submission.sh | bash -s -- <ping_url> [repo_dir]
+#
+#   Or download and run locally:
+#     chmod +x validate-submission.sh
+#     ./validate-submission.sh <ping_url> [repo_dir]
+#
+# Arguments:
+#   ping_url   Your HuggingFace Space URL (e.g. https://your-space.hf.space)
+#   repo_dir   Path to your repo (default: current directory)
+#
+# Examples:
+#   ./validate-submission.sh https://my-team.hf.space
+#   ./validate-submission.sh https://my-team.hf.space ./my-repo
+#
+
+set -uo pipefail
+
+DOCKER_BUILD_TIMEOUT=600
+if [ -t 1 ]; then
+  RED='\033[0;31m'
+  GREEN='\033[0;32m'
+  YELLOW='\033[1;33m'
+  BOLD='\033[1m'
+  NC='\033[0m'
+else
+  RED='' GREEN='' YELLOW='' BOLD='' NC=''
+fi
+
+run_with_timeout() {
+  local secs="$1"; shift
+  if command -v timeout &>/dev/null; then
+    timeout "$secs" "$@"
+  elif command -v gtimeout &>/dev/null; then
+    gtimeout "$secs" "$@"
+  else
+    "$@" &
+    local pid=$!
+    ( sleep "$secs" && kill "$pid" 2>/dev/null ) &
+    local watcher=$!
+    wait "$pid" 2>/dev/null
+    local rc=$?
+    kill "$watcher" 2>/dev/null
+    wait "$watcher" 2>/dev/null
+    return $rc
+  fi
+}
+
+portable_mktemp() {
+  local prefix="${1:-validate}"
+  mktemp "${TMPDIR:-/tmp}/${prefix}-XXXXXX" 2>/dev/null || mktemp
+}
+
+CLEANUP_FILES=()
+cleanup() { rm -f "${CLEANUP_FILES[@]+"${CLEANUP_FILES[@]}"}"; }
+trap cleanup EXIT
+
+PING_URL="${1:-}"
+REPO_DIR="${2:-.}"
+
+if [ -z "$PING_URL" ]; then
+  printf "Usage: %s <ping_url> [repo_dir]\n" "$0"
+  printf "\n"
+  printf "  ping_url   Your HuggingFace Space URL (e.g. https://your-space.hf.space)\n"
+  printf "  repo_dir   Path to your repo (default: current directory)\n"
+  exit 1
+fi
+
+if ! REPO_DIR="$(cd "$REPO_DIR" 2>/dev/null && pwd)"; then
+  printf "Error: directory '%s' not found\n" "${2:-.}"
+  exit 1
+fi
+PING_URL="${PING_URL%/}"
+export PING_URL
+PASS=0
+
+log()  { printf "[%s] %b\n" "$(date -u +%H:%M:%S)" "$*"; }
+pass() { log "${GREEN}PASSED${NC} -- $1"; PASS=$((PASS + 1)); }
+fail() { log "${RED}FAILED${NC} -- $1"; }
+hint() { printf "  ${YELLOW}Hint:${NC} %b\n" "$1"; }
+stop_at() {
+  printf "\n"
+  printf "${RED}${BOLD}Validation stopped at %s.${NC} Fix the above before continuing.\n" "$1"
+  exit 1
+}
+
+printf "\n"
+printf "${BOLD}========================================${NC}\n"
+printf "${BOLD}  OpenEnv Submission Validator${NC}\n"
+printf "${BOLD}========================================${NC}\n"
+log "Repo:     $REPO_DIR"
+log "Ping URL: $PING_URL"
+printf "\n"
+
+log "${BOLD}Step 1/3: Pinging HF Space${NC} ($PING_URL/reset) ..."
+
+CURL_OUTPUT=$(portable_mktemp "validate-curl")
+CLEANUP_FILES+=("$CURL_OUTPUT")
+HTTP_CODE=$(curl -s -o "$CURL_OUTPUT" -w "%{http_code}" -X POST \
+  -H "Content-Type: application/json" -d '{}' \
+  "$PING_URL/reset" --max-time 30 2>"$CURL_OUTPUT" || printf "000")
+
+if [ "$HTTP_CODE" = "200" ]; then
+  pass "HF Space is live and responds to /reset"
+elif [ "$HTTP_CODE" = "000" ]; then
+  fail "HF Space not reachable (connection failed or timed out)"
+  hint "Check your network connection and that the Space is running."
+  hint "Try: curl -s -o /dev/null -w '%%{http_code}' -X POST $PING_URL/reset"
+  stop_at "Step 1"
+else
+  fail "HF Space /reset returned HTTP $HTTP_CODE (expected 200)"
+  hint "Make sure your Space is running and the URL is correct."
+  hint "Try opening $PING_URL in your browser first."
+  stop_at "Step 1"
+fi
+
+log "${BOLD}Step 2/3: Running docker build${NC} ..."
+
+if ! command -v docker &>/dev/null; then
+  fail "docker command not found"
+  hint "Install Docker: https://docs.docker.com/get-docker/"
+  stop_at "Step 2"
+fi
+
+if [ -f "$REPO_DIR/Dockerfile" ]; then
+  DOCKER_CONTEXT="$REPO_DIR"
+elif [ -f "$REPO_DIR/server/Dockerfile" ]; then
+  DOCKER_CONTEXT="$REPO_DIR/server"
+else
+  fail "No Dockerfile found in repo root or server/ directory"
+  stop_at "Step 2"
+fi
+
+log "  Found Dockerfile in $DOCKER_CONTEXT"
+
+BUILD_OK=false
+BUILD_OUTPUT=$(run_with_timeout "$DOCKER_BUILD_TIMEOUT" docker build "$DOCKER_CONTEXT" 2>&1) && BUILD_OK=true
+
+if [ "$BUILD_OK" = true ]; then
+  pass "Docker build succeeded"
+else
+  fail "Docker build failed (timeout=${DOCKER_BUILD_TIMEOUT}s)"
+  printf "%s\n" "$BUILD_OUTPUT" | tail -20
+  stop_at "Step 2"
+fi
+
+log "${BOLD}Step 3/3: Running openenv validate${NC} ..."
+
+if ! command -v openenv &>/dev/null; then
+  fail "openenv command not found"
+  hint "Install it: pip install openenv-core"
+  stop_at "Step 3"
+fi
+
+VALIDATE_OK=false
+VALIDATE_OUTPUT=$(cd "$REPO_DIR" && openenv validate 2>&1) && VALIDATE_OK=true
+
+if [ "$VALIDATE_OK" = true ]; then
+  pass "openenv validate passed"
+  [ -n "$VALIDATE_OUTPUT" ] && log "  $VALIDATE_OUTPUT"
+else
+  fail "openenv validate failed"
+  printf "%s\n" "$VALIDATE_OUTPUT"
+  stop_at "Step 3"
+fi
+
+printf "\n"
+printf "${BOLD}========================================${NC}\n"
+printf "${GREEN}${BOLD}  All 3/3 checks passed!${NC}\n"
+printf "${GREEN}${BOLD}  Your submission is ready to submit.${NC}\n"
+printf "${BOLD}========================================${NC}\n"
+printf "\n"
+
+exit 0