feat: added ASBench annotation interface

.gitignore

@@ -205,4 +205,4 @@ cython_debug/
 # Marimo
 marimo/_static/
 marimo/_lsp/
-__marimo__/
+__marimo__/

annotation/annot_interface.py

@@ -0,0 +1,649 @@
+import streamlit as st
+import json
+from datetime import datetime
+
+
+st.set_page_config(
+    page_title="Annotation Interface",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+
+st.markdown("""
+<style>
+@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap');
+
+:root {
+    --bg: #0f1117;
+    --surface: #1a1d27;
+    --surface2: #22263a;
+    --border: #2e3350;
+    --accent: #4f8ef7;
+    --accent2: #7c5cbf;
+    --success: #3ecf8e;
+    --warning: #f5a623;
+    --danger: #e05c5c;
+    --text: #e8eaf0;
+    --text-muted: #9aa3bf;
+    --mono: 'JetBrains Mono', monospace;
+    --sans: 'Inter', sans-serif;
+}
+
+html, body, [class*="css"] {
+    font-family: var(--sans);
+    background-color: var(--bg);
+    color: var(--text);
+    font-size: 15px;
+}
+
+.stApp { background: var(--bg); }
+
+/* Header */
+.anno-header {
+    display: flex;
+    align-items: center;
+    gap: 16px;
+    padding: 28px 0 16px;
+    border-bottom: 1px solid var(--border);
+    margin-bottom: 28px;
+}
+.anno-title {
+    font-family: var(--sans);
+    font-size: 20px;
+    font-weight: 700;
+    color: var(--text);
+    letter-spacing: -0.3px;
+}
+.anno-subtitle {
+    font-size: 14px;
+    color: var(--text-muted);
+    font-weight: 400;
+}
+
+/* Task card */
+.query-card {
+    background: var(--surface);
+    border: 1px solid var(--border);
+    border-left: 4px solid var(--accent);
+    border-radius: 8px;
+    padding: 20px 24px;
+    margin-bottom: 12px;
+}
+.query-label {
+    font-family: var(--sans);
+    font-size: 11px;
+    font-weight: 600;
+    text-transform: uppercase;
+    letter-spacing: 1.2px;
+    color: var(--accent);
+    margin-bottom: 10px;
+}
+.query-text {
+    font-size: 18px;
+    font-weight: 400;
+    line-height: 1.75;
+    color: var(--text);
+}
+.golden-label {
+    display: inline-block;
+    background: rgba(245,166,35,0.12);
+    border: 1px solid rgba(245,166,35,0.35);
+    color: var(--warning);
+    font-size: 13px;
+    font-weight: 500;
+    padding: 4px 12px;
+    border-radius: 4px;
+    margin-top: 10px;
+}
+
+/* Progress bar */
+.progress-row {
+    display: flex;
+    align-items: center;
+    gap: 14px;
+    margin-bottom: 24px;
+}
+.progress-bar-outer {
+    flex: 1;
+    height: 5px;
+    background: var(--border);
+    border-radius: 3px;
+    overflow: hidden;
+}
+.progress-bar-inner {
+    height: 100%;
+    background: linear-gradient(90deg, var(--accent), var(--accent2));
+    border-radius: 3px;
+    transition: width 0.3s ease;
+}
+.progress-label {
+    font-size: 13px;
+    font-weight: 500;
+    color: var(--text-muted);
+    white-space: nowrap;
+}
+
+/* Score pills */
+.score-pill {
+    font-size: 13px;
+    font-weight: 600;
+    padding: 3px 12px;
+    border-radius: 4px;
+    white-space: nowrap;
+}
+.score-high { background: rgba(62,207,142,0.15); color: var(--success); border: 1px solid rgba(62,207,142,0.3); }
+.score-mid  { background: rgba(245,166,35,0.12); color: var(--warning); border: 1px solid rgba(245,166,35,0.3); }
+.score-low  { background: rgba(224,92,92,0.12);  color: var(--danger);  border: 1px solid rgba(224,92,92,0.3); }
+
+/* Reasoning box */
+.reasoning-box {
+    background: var(--surface2);
+    border: 1px solid var(--border);
+    border-radius: 6px;
+    padding: 16px 20px;
+    font-size: 16px;
+    font-weight: 400;
+    color: var(--text-muted);
+    line-height: 1.75;
+    margin-top: 8px;
+}
+.reasoning-box .box-label {
+    font-size: 11px;
+    font-weight: 600;
+    text-transform: uppercase;
+    letter-spacing: 1.2px;
+    color: var(--accent2);
+    margin-bottom: 10px;
+    display: block;
+}
+
+/* Agent response markdown text */
+.stMarkdown p, .stMarkdown li, .stMarkdown td {
+    font-size: 16px !important;
+    line-height: 1.75 !important;
+}
+
+/* Section labels */
+.section-label {
+    font-size: 11px;
+    font-weight: 600;
+    text-transform: uppercase;
+    letter-spacing: 1.2px;
+    color: var(--text-muted);
+    margin-bottom: 8px;
+    display: block;
+}
+.section-label-accent {
+    font-size: 11px;
+    font-weight: 600;
+    text-transform: uppercase;
+    letter-spacing: 1.2px;
+    color: var(--accent);
+    margin-bottom: 8px;
+    display: block;
+}
+
+/* Dividers */
+.section-divider {
+    border: none;
+    border-top: 1px solid var(--border);
+    margin: 20px 0;
+}
+
+/* Stats row */
+.stats-row {
+    display: flex;
+    gap: 12px;
+    flex-wrap: wrap;
+    margin-bottom: 20px;
+}
+.stat-box {
+    background: var(--surface);
+    border: 1px solid var(--border);
+    border-radius: 8px;
+    padding: 14px 20px;
+    text-align: center;
+    min-width: 110px;
+}
+.stat-val {
+    font-size: 24px;
+    font-weight: 700;
+    color: var(--accent);
+    line-height: 1.2;
+}
+.stat-lbl {
+    font-size: 12px;
+    font-weight: 500;
+    color: var(--text-muted);
+    margin-top: 4px;
+    text-transform: uppercase;
+    letter-spacing: 0.8px;
+}
+
+/* Nav buttons */
+.stButton > button {
+    background: var(--surface) !important;
+    color: var(--text) !important;
+    border: 1px solid var(--border) !important;
+    border-radius: 6px !important;
+    font-family: var(--sans) !important;
+    font-size: 14px !important;
+    font-weight: 500 !important;
+    transition: all 0.15s ease !important;
+}
+.stButton > button:hover {
+    border-color: var(--accent) !important;
+    color: var(--accent) !important;
+    background: rgba(79,142,247,0.07) !important;
+}
+
+/* Selectbox / textarea overrides */
+.stSelectbox > div > div, .stTextArea > div > textarea {
+    background: var(--surface2) !important;
+    border-color: var(--border) !important;
+    color: var(--text) !important;
+    font-family: var(--sans) !important;
+    font-size: 14px !important;
+}
+
+/* Expander styling */
+details > summary {
+    font-family: var(--sans) !important;
+    font-size: 14px !important;
+    font-weight: 500 !important;
+    color: var(--text) !important;
+}
+details > summary:hover {
+    color: var(--accent) !important;
+}
+
+/* Response time mono */
+.resp-time {
+    font-family: var(--mono);
+    font-size: 13px;
+    color: var(--text-muted);
+}
+
+/* Hide default streamlit elements */
+#MainMenu, footer, header { visibility: hidden; }
+.block-container { padding-top: 1rem; max-width: 1200px; }
+</style>
+""", unsafe_allow_html=True)
+
+
+class AnnotationInterface:
+    def __init__(self):
+        self._init_session_state()
+
+    def _init_session_state(self):
+        if "current_query_index" not in st.session_state:
+            st.session_state.current_query_index = 0
+        if "annotations" not in st.session_state:
+            st.session_state.annotations = {}
+        if "grouped_data" not in st.session_state:
+            st.session_state.grouped_data = {}
+        if "query_ids" not in st.session_state:
+            st.session_state.query_ids = []
+        if "expanded_responses" not in st.session_state:
+            st.session_state.expanded_responses = set()
+
+    @staticmethod
+    def load_jsonl(file):
+        data = []
+        content = file.getvalue().decode("utf-8")
+        for line in content.strip().split("\n"):
+            if line.strip():
+                data.append(json.loads(line))
+        return data
+
+    @staticmethod
+    def process_data(queries_list, responses_list, labels_list):
+        queries_dict = {list(q.keys())[0]: q[list(q.keys())[0]] for q in queries_list}
+        responses_dict = {list(r.keys())[0]: r[list(r.keys())[0]] for r in responses_list}
+        labels_dict = {list(l.keys())[0]: l[list(l.keys())[0]] for l in labels_list}
+
+        all_query_ids = set(queries_dict.keys()) & set(responses_dict.keys()) & set(labels_dict.keys())
+
+        grouped = {}
+        for qid in sorted(all_query_ids):
+            query_text = queries_dict[qid][0]
+            golden_label = queries_dict[qid][1] if type(queries_dict[qid][1]) == str else None
+            responses = responses_dict.get(qid, {})
+            labels = labels_dict.get(qid, {})
+
+            items = []
+            for agent_id in responses.keys():
+                if agent_id in labels:
+                    response_text, response_time = responses[agent_id]
+                    llm_score, llm_reasoning = labels[agent_id]
+                    items.append({
+                        "query_id": qid,
+                        "agent_id": agent_id,
+                        "query": query_text,
+                        "golden_label": golden_label,
+                        "response": response_text,
+                        "response_time": response_time,
+                        "llm_score": llm_score,
+                        "llm_reasoning": llm_reasoning,
+                    })
+
+            if items:
+                grouped[qid] = {
+                    "query": query_text,
+                    "golden_label": golden_label,
+                    "responses": items,
+                }
+
+        return grouped
+
+    @staticmethod
+    def score_class(score):
+        if score >= 4:
+            return "score-high"
+        elif score >= 3:
+            return "score-mid"
+        return "score-low"
+
+    def get_annotation_key(self, query_id, agent_id):
+        return f"{query_id}___{agent_id}"
+
+    def count_annotated_for_query(self, query_id, responses):
+        return sum(
+            1 for r in responses
+            if self.get_annotation_key(query_id, r["agent_id"]) in st.session_state.annotations
+        )
+
+    def count_total_annotated(self):
+        return len(st.session_state.annotations)
+
+    def count_total_responses(self):
+        return sum(len(v["responses"]) for v in st.session_state.grouped_data.values())
+
+    @staticmethod
+    def export_annotations():
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f"human_annotations_{timestamp}.json"
+        output_data = {
+            "timestamp": timestamp,
+            "annotations": list(st.session_state.annotations.values()),
+        }
+        return json.dumps(output_data, indent=2), filename
+
+    def load_continuation(self, file):
+        """Parse a previous annotations JSON export and return a dict keyed by query_id___agent_id."""
+        content = file.getvalue().decode("utf-8")
+        data = json.loads(content)
+        annotations = {}
+        loaded = 0
+        for item in data.get("annotations", []):
+            qid = item.get("query_id")
+            aid = item.get("agent_id")
+            if qid and aid:
+                key = self.get_annotation_key(qid, aid)
+                annotations[key] = {
+                    "query_id": qid,
+                    "agent_id": aid,
+                    "query": item.get("query", ""),
+                    "llm_score": item.get("llm_score"),
+                    "llm_reasoning": item.get("llm_reasoning", ""),
+                    "human_score": item.get("human_score", 3),
+                    "explanation": item.get("explanation", ""),
+                }
+                loaded += 1
+        return annotations, loaded
+
+    def render_file_upload(self):
+        st.markdown('<div class="anno-header"><span class="anno-title">Annotation Interface</span><span class="anno-subtitle"></span></div>', unsafe_allow_html=True)
+
+        st.markdown("### Upload Data Files")
+        col1, col2, col3 = st.columns(3)
+        with col1:
+            queries_file = st.file_uploader("Tasks (JSONL)", type=["jsonl"], key="queries")
+        with col2:
+            responses_file = st.file_uploader("Agent Responses (JSONL)", type=["jsonl"], key="responses")
+        with col3:
+            labels_file = st.file_uploader("LLM Judge Labels (JSONL)", type=["jsonl"], key="labels")
+
+        st.markdown('<hr class="section-divider">', unsafe_allow_html=True)
+        st.markdown("### Resume from Previous Session  *(optional)*")
+        st.markdown(
+            '<span style="font-size:13px;color:var(--text-muted)">Upload a previously downloaded annotations JSON file to pre-fill your prior scores and continue where you left off.</span>',
+            unsafe_allow_html=True,
+        )
+        continuation_file = st.file_uploader(
+            "Continuation file (JSON)",
+            type=["json"],
+            key="continuation",
+            help="A human_annotations_*.json file downloaded from a previous session.",
+        )
+
+        if continuation_file:
+            try:
+                preview_data = json.loads(continuation_file.getvalue().decode("utf-8"))
+                n = len(preview_data.get("annotations", []))
+                ts = preview_data.get("timestamp", "unknown")
+                st.markdown(
+                    f'<div style="background:rgba(79,142,247,0.08);border:1px solid rgba(79,142,247,0.25);border-radius:6px;padding:12px 16px;font-size:13px;color:var(--text-muted);margin-top:8px;">'
+                    f'<strong style="color:var(--accent)">✓ Continuation file detected</strong> &nbsp;·&nbsp; '
+                    f'Saved on <code>{ts}</code> &nbsp;·&nbsp; <strong>{n}</strong> prior annotation(s) will be restored.'
+                    f'</div>',
+                    unsafe_allow_html=True,
+                )
+            except Exception:
+                st.warning("Could not parse continuation file — it will be ignored.")
+                continuation_file = None
+
+        st.markdown('<div style="height:8px"></div>', unsafe_allow_html=True)
+
+        if queries_file and responses_file and labels_file:
+            if st.button("Load Data →"):
+                with st.spinner("Processing files..."):
+                    queries_list = self.load_jsonl(queries_file)
+                    responses_list = self.load_jsonl(responses_file)
+                    labels_list = self.load_jsonl(labels_file)
+                    grouped = self.process_data(queries_list, responses_list, labels_list)
+                    st.session_state.grouped_data = grouped
+                    st.session_state.query_ids = list(grouped.keys())
+                    st.session_state.current_query_index = 0
+
+                    restored = 0
+                    if continuation_file:
+                        try:
+                            continuation_file.seek(0)
+                            prior_annotations, restored = self.load_continuation(continuation_file)
+                            st.session_state.annotations = prior_annotations
+                        except Exception as e:
+                            st.warning(f"Could not load continuation file: {e}")
+                            st.session_state.annotations = {}
+                    else:
+                        st.session_state.annotations = {}
+
+                    total_resp = sum(len(v["responses"]) for v in grouped.values())
+                    msg = f"Loaded {len(grouped)} queries with {total_resp} total responses."
+                    if restored:
+                        msg += f" Restored **{restored}** prior annotation(s) from continuation file."
+                    st.success(msg)
+                    st.rerun()
+        else:
+            st.info("Upload the three required JSONL files to begin annotation.")
+
+    def render_main(self):
+        query_ids = st.session_state.query_ids
+        grouped = st.session_state.grouped_data
+        total_queries = len(query_ids)
+        qidx = st.session_state.current_query_index
+        qid = query_ids[qidx]
+        query_data = grouped[qid]
+
+        total_responses = self.count_total_responses()
+        total_annotated = self.count_total_annotated()
+
+        # Header
+        st.markdown(
+            f'<div class="anno-header">'
+            f'<span class="anno-title">Annotation Interface</span>'
+            f'<span class="anno-subtitle">Task evaluation workbench</span>'
+            f'</div>',
+            unsafe_allow_html=True,
+        )
+
+        # Stats row
+        q_annotated = self.count_annotated_for_query(qid, query_data["responses"])
+        q_total = len(query_data["responses"])
+        st.markdown(
+            f'<div class="stats-row">'
+            f'<div class="stat-box"><div class="stat-val">{qidx + 1}/{total_queries}</div><div class="stat-lbl">Task</div></div>'
+            f'<div class="stat-box"><div class="stat-val">{q_annotated}/{q_total}</div><div class="stat-lbl">Annotated</div></div>'
+            f'<div class="stat-box"><div class="stat-val">{total_annotated}/{total_responses}</div><div class="stat-lbl">Total Done</div></div>'
+            f'</div>',
+            unsafe_allow_html=True,
+        )
+
+        # Progress bar
+        pct = (total_annotated / total_responses * 100) if total_responses else 0
+        st.markdown(
+            f'<div class="progress-row">'
+            f'<div class="progress-bar-outer"><div class="progress-bar-inner" style="width:{pct:.1f}%"></div></div>'
+            f'<span class="progress-label">{pct:.0f}% complete</span>'
+            f'</div>',
+            unsafe_allow_html=True,
+        )
+
+        # Task card
+        golden_html = ""
+        if query_data["golden_label"]:
+            golden_html = f'<div class="golden-label">⚑ GOLDEN LABEL: {query_data["golden_label"]}</div>'
+        st.markdown(
+            f'<div class="query-card">'
+            f'<div class="query-label">Task · {qid}</div>'
+            f'<div class="query-text">{query_data["query"]}</div>'
+            f'{golden_html}'
+            f'</div>',
+            unsafe_allow_html=True,
+        )
+
+        st.markdown('<hr class="section-divider">', unsafe_allow_html=True)
+
+        # Responses
+        st.markdown(f'<div class="section-label">{q_total} Responses — expand to annotate</div>', unsafe_allow_html=True)
+
+        for i, resp in enumerate(query_data["responses"]):
+            anno_key = self.get_annotation_key(qid, resp["agent_id"])
+            existing = st.session_state.annotations.get(anno_key, {})
+            has_annotation = bool(existing)
+            llm_score = resp["llm_score"]
+            sc = self.score_class(llm_score)
+
+            # Expander label
+            short_agent = resp["agent_id"][:40] + "…" if len(resp["agent_id"]) > 40 else resp["agent_id"]
+            label = f"Response {i+1:02d} — {short_agent}  ·  LLM: {llm_score}/5{'  ·  ✓ annotated' if has_annotation else ''}"
+
+            with st.expander(label, expanded=False):
+                # LLM info row
+                col_score, col_time = st.columns([1, 2])
+                with col_score:
+                    st.markdown(f'<div style="margin-bottom:8px"><span class="section-label">LLM Score</span><span class="score-pill {sc}" style="font-size:15px;padding:5px 16px">{llm_score}/5</span></div>', unsafe_allow_html=True)
+                with col_time:
+                    st.markdown(f'<div style="margin-bottom:8px"><span class="section-label">Response Time</span><span class="resp-time">{resp["response_time"]:.2f}s</span></div>', unsafe_allow_html=True)
+
+                st.markdown(f'<div class="reasoning-box"><span class="box-label">LLM Reasoning</span>{resp["llm_reasoning"]}</div>', unsafe_allow_html=True)
+
+                st.markdown('<div style="height:14px"></div>', unsafe_allow_html=True)
+
+                st.markdown('<span class="section-label">Agent Response</span>', unsafe_allow_html=True)
+                st.markdown(resp["response"])
+
+                st.markdown('<hr class="section-divider">', unsafe_allow_html=True)
+
+                # Annotation inputs
+                st.markdown('<span class="section-label-accent">Your Annotation</span>', unsafe_allow_html=True)
+
+                a_col1, a_col2 = st.columns([1, 3])
+                with a_col1:
+                    human_score = st.selectbox(
+                        "Score (1–5)",
+                        options=[1, 2, 3, 4, 5],
+                        index=existing.get("human_score", 3) - 1,
+                        key=f"score_{anno_key}",
+                    )
+                with a_col2:
+                    human_explanation = st.text_area(
+                        "Explanation",
+                        value=existing.get("explanation", ""),
+                        height=90,
+                        placeholder="Your reasoning for the score…",
+                        key=f"expl_{anno_key}",
+                    )
+
+                if st.button("Save annotation", key=f"save_{anno_key}"):
+                    st.session_state.annotations[anno_key] = {
+                        "query_id": qid,
+                        "agent_id": resp["agent_id"],
+                        "query": resp["query"],
+                        "llm_score": llm_score,
+                        "llm_reasoning": resp["llm_reasoning"],
+                        "human_score": human_score,
+                        "explanation": human_explanation,
+                    }
+                    st.rerun()
+
+        st.markdown('<hr class="section-divider">', unsafe_allow_html=True)
+
+        # Navigation
+        nav_col1, nav_col2, nav_col3, nav_col4 = st.columns([1, 1, 1, 2])
+        with nav_col1:
+            if st.button("← Previous", disabled=qidx == 0, use_container_width=True):
+                st.session_state.current_query_index -= 1
+                st.rerun()
+        with nav_col2:
+            if st.button("Next →", disabled=qidx >= total_queries - 1, use_container_width=True):
+                st.session_state.current_query_index += 1
+                st.rerun()
+        with nav_col3:
+            jump = st.number_input("Go to query #", min_value=1, max_value=total_queries, value=qidx + 1, label_visibility="collapsed")
+            if jump - 1 != qidx:
+                st.session_state.current_query_index = jump - 1
+                st.rerun()
+        with nav_col4:
+            if total_annotated > 0:
+                json_str, filename = self.export_annotations()
+                st.download_button(
+                    label=f"⬇ Download all annotations ({total_annotated})",
+                    data=json_str,
+                    file_name=filename,
+                    mime="application/json",
+                    use_container_width=True,
+                )
+
+        # Sidebar
+        with st.sidebar:
+            st.markdown("### ⚙️ Settings")
+            if st.button("Load new files"):
+                st.session_state.grouped_data = {}
+                st.session_state.query_ids = []
+                st.session_state.annotations = {}
+                st.session_state.current_query_index = 0
+                st.rerun()
+
+            st.markdown("---")
+            st.markdown("### 📊 Task Progress")
+            for i, qid_s in enumerate(query_ids):
+                qd = grouped[qid_s]
+                done = self.count_annotated_for_query(qid_s, qd["responses"])
+                total = len(qd["responses"])
+                marker = "✓" if done == total else ("◑" if done > 0 else "○")
+                label_s = f"{marker} [{i+1}] {qid_s[:20]}… ({done}/{total})"
+                if st.button(label_s, key=f"nav_{qid_s}", use_container_width=True):
+                    st.session_state.current_query_index = i
+                    st.rerun()
+
+    def run(self):
+        if not st.session_state.grouped_data:
+            self.render_file_upload()
+        else:
+            self.render_main()
+
+
+if __name__ == "__main__":
+    app = AnnotationInterface()
+    app.run()

annotation/annotation_tasks/tasks.jsonl

@@ -0,0 +1,50 @@
+{"g:0": ["Of the cities within the United States where U.S. presidents were born, which two are the farthest apart from the westernmost to the easternmost going east, giving the city names only? Give them to me in alphabetical order, in a comma-separated list", "Braintree, Honolulu"]}
+{"g:1": ["In Unlambda, what exact charcter or text needs to be added to correct the following code to output \"For penguins\"? If what is needed is a character, answer with the name of the character. If there are different names for the character, use the shortest. The text location is not needed. Code:\n\n`r```````````.F.o.r. .p.e.n.g.u.i.n.si", "backtick"]}
+{"g:2": ["A 5-man group made up of one tank, one healer, and three DPS is doing a dungeon that was just released in World of Warcraft. Two are plate wearers and two are cloth wearers. At the final boss, both the tank and the healer are casting holy spells. Ice and fire are being used, each one by a different DPS. A bear from the group is attacking the boss. Metamorphosis is cast. The Kilt of the Forgotten One drops as loot, but no one can use it. If all classes were using their class abilities and all classes are unique, what are the five classes in the group in alphabetical order separated by commas?", "Death Knight, Hunter, Paladin, Priest, Warlock"]}
+{"g:3": [".rewsna eht sa \"tfel\" drow eht fo etisoppo eht etirw ,ecnetnes siht dnatsrednu uoy fI", "Right"]}
+{"g:4": ["In Audre Lorde\u2019s poem \u201cFather Son and Holy Ghost\u201d, what is the number of the stanza in which some lines are indented?", "2"]}
+{"g:5": ["In terms of geographical distance between capital cities, which 2 countries are the furthest from each other within the ASEAN bloc according to wikipedia? Answer using a comma separated list, ordering the countries by alphabetical order.", "Indonesia, Myanmar"]}
+{"g:6": ["What two-word type of model did Manash Pratim Kashyap's and PS Fader's studies in customer retention studies published during 2018-2019 have in common (no punctuation)?", "beta geometric"]}
+{"g:7": ["It is 1999. Before you party like it is 1999, please assist me in settling a bet.\n\nFiona Apple and Paula Cole released albums prior to 1999. Of these albums, which didn't receive a letter grade from Robert Christgau? Provide your answer as a comma delimited list of album titles, sorted alphabetically.", "Harbinger, Tidal"]}
+{"g:8": ["You are a telecommunications engineer who wants to build cell phone towers on a stretch of road. In the reference file is a layout of the road and nearby houses. Each dash, \"-\", is a marker indicating a mile. Each capital H indicates a house located next to a mile marker, appearing above or below the stretch of road. Each cell phone tower can cover houses located next to the road within a 4-mile radius. Find the minimum number of cell phone towers needed to cover all houses next to the road. Your answer should be a positive numerical integer value.", "3"]}
+{"g:9": ["It's May 2023, and I'm about to drive across the U.S. from California to Maine. I always recycle my water bottles at the end of a trip, and I drink 5 12-ounce water bottles for every 100 miles I travel, rounded to the nearest 100. Assuming I follow I-40 from Los Angeles to Cincinnati, then take I-90 from Cincinnati to Augusta, how many dollars will I get back according to Wikipedia?", "8"]}
+{"t:0": ["Draft a legally compliant performance improvement plan (PIP) and termination checklist for an underperforming salaried customer-support employee in Jaipur, Rajasthan, India, at a 60-person tech company. Include scripts, a 30/60/90-day timeline, and documentation templates for meetings, warnings, and final separation.", ["agt:openaiagents:402f63@v1.1"]]}
+{"t:1": ["Create an IP risk checklist and filing roadmap (patents, trademarks, trade secrets) for a new product: \u201cMantaTest,\u201d an open\u2011source GPL C# OOP neural\u2011network framework (http://www.mantatest.com/). Include priority deadlines, steps for US first filing/PCT, trademark clearance/filing, trade secret controls, and a basic IP portfolio tracker template.", ["agt:openaiagents:0fc77b@v1.1"]]}
+{"t:2": ["Build a step-by-step checklist (with required forms and evidence) to apply for VA health care and verify eligibility, including how to use VA.gov and what to do if denied.", ["agt:openaiagents:fdd41a@v1.1"]]}
+{"t:3": ["Given this job description for a Records & Information Management role (ICRM-related) from http://www.gamerevolution.com/oldsite/games/dreamcast/action/maken_x.htm, rewrite my resume summary and bullets for ATS and role priorities. My background: 5 years records management, retention schedules, eDiscovery, SharePoint, compliance, process improvement. Note key hiring-manager priorities.", ["agt:agentainetwork:635a65@v1.1"]]}
+{"t:4": ["Make a country-by-country briefing comparing how India, Japan, and Brazil frame the UN Security Council reform debate, with key cultural/historical context and recent diplomacy timeline.", ["agt:openaiagents:82bfe6@v1.1"]]}
+{"t:5": ["Create a triage decision tree for my dog\u2019s vomiting and diarrhea (include red flags, at-home steps, and when to seek emergency care vs schedule a vet visit).", ["agt:openaiagents:f3b5e9@v1.1"]]}
+{"t:6": ["Can you help me map my anxiety triggers by asking me a few questions, then summarize patterns and give a personalized 7-day coping plan with daily check-ins?", ["agt:openaiagents:f94d6c@v1.1"]]}
+{"t:7": ["Create a HACCP-style risk assessment for my homemade refrigerated chicken salad. Ingredients: cooked diced chicken breast, mayonnaise, celery, red onion, Dijon mustard, lemon juice, salt, pepper. Process: cook chicken to 74\u00b0C/165\u00b0F, cool, dice, mix with ingredients, portion into sealed containers, refrigerate, serve. Identify hazards, critical control points, critical limits, monitoring/corrective actions, and estimate safe shelf life at \u22644\u00b0C/40\u00b0F.", ["agt:openaiagents:7d97ac@v1.1"]]}
+{"t:8": ["Review my SaaS services agreement for AcmeCloud (B2B subscription software) with Customer XYZ (US-based). Provide prioritized redlines and a negotiation plan (fallback positions, tradeoffs, and email wording) to reduce liability and tighten payment and termination terms. Assume current contract includes broad indemnity, uncapped damages, net-60, and vague termination.", ["agt:openaiagents:88512b@v1.1"]]}
+{"t:9": ["Create a 90-day retention plan for our 25-person startup, including manager actions, stay-interview questions, and metrics to track voluntary turnover monthly.", ["agt:openaiagents:e373b2@v1.1"]]}
+{"t:10": ["Set up a Zapier automation: new Typeform lead \u2192 enrich with Clearbit \u2192 create/update HubSpot contact + deal \u2192 post to Slack, and alert me if enrichment fails.", ["agt:openaiagents:a8d6ea@v1.1"]]}
+{"t:11": ["I\u2019m a Sales Operations Manager at a 200-person B2B SaaS company using Salesforce, HubSpot, Outreach, and Google Sheets. Analyze my weekly tasks (lead routing, pipeline hygiene, forecast reporting, territory changes, deal desk approvals, dashboard upkeep, enablement, renewals handoffs) and bundle into AI-assisted workflows with automations, suggested tools, time-saved estimates, and a 30/60/90-day rollout plan.", ["agt:agentainetwork:abca0b@v1.1"]]}
+{"t:12": ["Design a Make.com scenario that auto-replies to new Gmail emails using a Google Sheets FAQ lookup, includes error handling, and outputs a step-by-step module blueprint.", ["agt:openaiagents:7ac082@v1.1"]]}
+{"t:13": ["Audit February 2026 cross-channel marketing spend and ROI for Acme D2C (Google Search/Shopping, Meta, TikTok, email, affiliates). Provide budget reallocation, 30-day optimization plan with KPI targets, team owners, and an analytics tracking checklist (UTMs, GA4, pixels/CAPI, offline conversions, attribution).", ["agt:openaiagents:d2d940@v1.1"]]}
+{"t:14": ["Find current deals on premium project-management tools under $10/month (e.g., Notion, Asana alternatives) and list the best offers with links and renewal prices.", ["agt:agentainetwork:6e79ea@v1.1"]]}
+{"t:15": ["Create a shareable daily \u201cVisitor Sign-in\u201d form with required fields, a liability acknowledgment checkbox, and an auto-generated timestamp; include a simple admin review checklist.", ["agt:openaiagents:ec899f@v1.1"]]}
+{"t:16": ["I\u2019m moving into a small apartment\u2014give me a 30\u2011minute, low-cost plan (with materials list) to organize my kitchen and reduce food waste using simple DIY hacks.", ["agt:openaiagents:2d1a05@v1.1"]]}
+{"t:17": ["Turn my raw meeting notes (below) from a 1-hour project sync (Mar 6, 2026, 10:00\u201311:00 AM ET; attendees: Alex Chen, Priya Patel, Jordan Lee) into polished minutes with: agenda sections, key discussion points, decisions, action items (owner + due date), parking lot, and a 3\u20135 sentence email-ready recap. Notes: [PASTE NOTES HERE].", ["agt:agentainetwork:b62ccf@v1.1"]]}
+{"t:18": ["I\u2019m meeting Sarah Chen (VP Partnerships at a B2B SaaS procurement platform) tomorrow to discuss a potential integration/reseller deal. Using professional insights from this video https://www.youtube.com/watch?v=B8TOz25ctGw, draft a 30\u2011minute tailored meeting agenda, 6 rapport starters, and 5 likely objections with concise responses.", ["agt:agentainetwork:9d9c3f@v1.1"]]}
+{"t:19": ["Given this job description for an ATS-screened role at ImproveNet (radiodata.com) in the home appliances market, identify what the company and hiring manager likely prioritize, then rewrite my current resume bullet points to match the required skills/keywords. Assume I\u2019m a mid-level product marketing manager with 5 years\u2019 experience in consumer appliances.", ["agt:agentainetwork:635a65@v1.1"]]}
+{"t:20": ["In Figma, build a reusable color + typography token system (light/dark) and show the exact token names plus CSS variable equivalents I can paste into my codebase.", ["agt:openaiagents:207933@v1.1"]]}
+{"t:21": ["Create a Figma-ready wireframe (screens + components list) for a mobile inventory system: login, dashboard, scan item, add/edit item, stock adjustments, audit log, and alerts.", ["agt:openaiagents:abb118@v1.1"]]}
+{"t:22": ["Create a low\u2011fidelity Figma wireframe for a mobile checkout flow (cart \u2192 address \u2192 payment \u2192 review), with frame list, component labels, and layout specs.", ["agt:openaiagents:e4fbaf@v1.1"]]}
+{"t:23": ["Design a cohesive set of 12 minimalist space-themed app icons as a single downloadable PNG grid, with consistent stroke/size and transparent background.", ["agt:openaiagents:e624ec@v1.1"]]}
+{"t:24": ["Create PlantUML for a data flow diagram of an online food ordering system (customer, restaurant, payment gateway), including processes, data stores, and labeled data flows.", ["agt:openaiagents:9eb1e6@v1.1"]]}
+{"t:25": ["Create a 1:1 neon cyberpunk coffee logo image, then save it and give me a shareable link plus downloadable PNG and SVG versions.", ["agt:openaiagents:5f952a@v1.1"]]}
+{"t:26": ["Convert this Python function into a flowchart (Mermaid) showing decision branches and loops, and label each node with the corresponding code lines:\n\n```python\ndef validate_password(pw):\n    if len(pw) < 8:\n        return False\n    has_upper = False\n    has_digit = False\n    for ch in pw:\n        if ch.isupper():\n            has_upper = True\n        elif ch.isdigit():\n            has_digit = True\n        if has_upper and has_digit:\n            return True\n    return False\n```", ["agt:openaiagents:61ca34@v1.1"]]}
+{"t:27": ["Create 3 ultra-detailed FLUX.1 photoreal cinematic prompts for a rain-soaked neon alley detective scene, with camera/lens, lighting, composition, wardrobe, and negative prompts.", ["agt:openaiagents:627242@v1.1"]]}
+{"t:28": ["Convert my Figma design (assume a SaaS dashboard UI with header, sidebar, buttons, form inputs, cards, table, and modal) into a responsive React + TypeScript component library using Tailwind. Include exported SVG assets, folder/file structure, and paste-ready code examples for each component.", ["agt:openaiagents:3e180b@v1.1"]]}
+{"t:29": ["Convert a Figma frame screenshot of a modern landing-page hero (top nav with logo and 4 links, primary CTA button, left-aligned headline/subtext, and right-side image placeholder) into responsive semantic HTML/CSS with matching spacing/colors and hover states. Output a single clean code snippet.", ["agt:openaiagents:f111b6@v1.1"]]}
+{"t:30": ["I have two photos of a cheek rash (photo A taken today, photo B taken 3 days ago). Based only on typical visual changes, estimate likely causes (e.g., contact dermatitis, acne, eczema, infection), list red-flag symptoms needing urgent care, and give a 7-day home-care plan including step-by-step patch-testing for new products.", ["agt:openaiagents:27b7a3@v1.1"]]}
+{"t:31": ["Create a step-by-step triage checklist and 2-week treatment plan for a suspected eczema flare (adult), including OTC options, red flags, and when to book dermatology follow-up.", ["agt:openaiagents:673673@v1.1"]]}
+{"t:32": ["Create a 2-week nail recovery plan for brittle, peeling nails after gel removal\u2014daily steps, product checklist, and hygiene rules; include a simple progress tracker.", ["agt:openaiagents:2dc63f@v1.1"]]}
+{"t:33": ["Using my uploaded face selfie (I\u2019m a 28-year-old with combination skin, mild acne on chin, visible pores on nose, some redness on cheeks), analyze likely skin concerns and create a morning/night skincare checklist. Recommend 3 budget and 3 high-end products tailored to these concerns, with brief usage notes.", ["agt:openaiagents:7c1ab0@v1.1"]]}
+{"t:34": ["Create a Wonyoungism-style morning-to-night schedule with skincare layering steps (AM/PM), a beginner workout, and 5 daily affirmations tailored to oily, acne-prone skin and school days.", ["agt:openaiagents:005d0e@v1.1"]]}
+{"t:35": ["Create a 7-day weight-loss meal plan (daily calories + macros) and a grocery list using these details: 32-year-old female, 5'6\", 170 lb, goal 1 lb/week loss; strength trains 3x/week + 8k steps/day; current diet: oatmeal, chicken/rice, salads, snacks; must-haves: eggs, Greek yogurt, chicken, salmon, quinoa, spinach; lactose-sensitive.", ["agt:openaiagents:bd9a01@v1.1"]]}
+{"t:36": ["Using current guidelines and key trials, draft a SOAP note and evidence-based treatment plan for an adult with chronic insomnia, including CBT-I steps and medication options.", ["agt:openaiagents:b1415c@v1.1"]]}
+{"t:37": ["Create an evidence-based differential and stepwise workup for acute hypoxemic respiratory failure with bilateral infiltrates, including ventilation strategies, meds, contraindications, and patient handout points.", ["agt:openaiagents:9334dd@v1.1"]]}
+{"t:38": ["Create a 2-week supplement-and-herbal plan for better sleep and lower stress, including dosages, timing, safety interactions, and a simple shopping list.", ["agt:openaiagents:74a116@v1.1"]]}
+{"t:39": ["I have a headache, stiff neck, and fever since last night\u2014can you triage me with a red-flag checklist and tell me whether I should go to urgent care or ER today?", ["agt:openaiagents:659375@v1.1"]]}