import React, { useState } from "react";
import ReactPlayer from "react-player";
import axios from "axios";
import AxisEvaluation from "./components/AxisEvaluation";
import initialVideos from "./videoData";
const API_BASE_URL = (
import.meta.env.VITE_API_BASE_URL ||
(import.meta.env.DEV ? "http://localhost:8000" : "")
).replace(/\/$/, "");
// Calibration example shown on the instructions page before the main task.
// Update this index once the example video has been chosen.
const EXAMPLE_VIDEO_INDEX = 0;
const evaluationAxes = [
{
id: "audio_content",
title: "Audio Content",
summary: "Evaluate each required sound or speech item independently.",
details: [
"Label every listed audio item separately, such as a slap sound, clap sound, spoken word, silence, or absence of background music.",
"Mark No for a specific item if that sound is missing, replaced by the wrong sound, or contradicted by extra audio.",
"Do not judge visual correctness here unless it affects what can be heard."
]
},
{
id: "visual_content",
title: "Visual Content",
summary: "Evaluate each required visible object, gesture, person, or action independently.",
details: [
"Label every listed visual item separately, including which hand/body part is used, what gesture/action occurs, and what scene elements are present.",
"Mark No for a specific item if it is missing, replaced, unclear, or performed by the wrong person/body part.",
"Do not judge sound or timing here unless it affects what can be seen."
]
},
{
id: "audio_visual_sync",
title: "Audio-Visual Synchronization",
summary: "Evaluate whether each audio event is synchronized with its corresponding visual event.",
details: [
"Only evaluate an audio-visual pair when the corresponding audio item and visual item are both correct enough to compare.",
"Mark No if the sound clearly leads or lags the visible event, or if a sound has no matching visual cause.",
"Use slower playback when synchronization is hard to judge."
]
},
{
id: "temporal_relation",
title: "Temporal Relation",
summary: "Evaluate requested ordering or temporal relationships separately from counts.",
details: [
"Only evaluate temporal relations when the relevant Audio Content, Visual Content, and Audio-Visual Synchronization components are correct enough to support the judgment.",
"Check requested relations such as before/after order, simultaneous events, synchronization as a temporal relation, pauses, or ending state.",
"Do not use this axis to judge how many times an event occurs; use Temporal Count for that."
]
},
{
id: "temporal_count",
title: "Temporal Count",
summary: "Enter how many times each requested event actually occurs in the video.",
details: [
"Only count an event when the relevant Audio Content, Visual Content, Audio-Visual Synchronization, and Temporal Relation components are correct enough to identify it.",
"Count each occurrence of the listed event and type the total number you observe.",
"Enter 0 if the event does not occur at all."
]
},
{
id: "constraint_following",
title: "Constraint Following",
summary: "Check the non-core prompt constraints as a single overall judgment.",
details: [
"Check scene, setting, camera, shot type, people count, subtitles or on-screen text, background music, extra sounds, extra speech, extra gestures or actions, and stillness after the required events.",
"Mark No if the clip violates any of these constraints stated in the prompt.",
"Use this as a final overall check after judging the more specific axes above."
]
}
];
const createEmptyAnnotations = (video) =>
Object.fromEntries(
evaluationAxes.map((axis) => [
axis.id,
{
items: Object.fromEntries(
(video.evaluationItems[axis.id] || []).map((item) => [
item.id,
{ label: "", rationale: "" }
])
)
}
])
);
export default function App() {
const [hasStarted, setHasStarted] = useState(false);
const [understandsInstructions, setUnderstandsInstructions] = useState(false);
const [showExample, setShowExample] = useState(false);
const [exampleCompleted, setExampleCompleted] = useState(false);
const [playbackRate, setPlaybackRate] = useState(1);
const [videos, setVideos] = useState(initialVideos);
const [currentIndex, setCurrentIndex] = useState(0);
const currentVideo = videos[currentIndex];
const [annotator, setAnnotator] = useState("");
const [annotations, setAnnotations] = useState(() => createEmptyAnnotations(currentVideo));
const updateAxisItem = (axisId, itemId, field, value) => {
setAnnotations((prev) => ({
...prev,
[axisId]: {
...prev[axisId],
items: {
...prev[axisId].items,
[itemId]: {
...prev[axisId].items[itemId],
[field]: value
}
}
}
}));
};
const saveAnnotations = async () => {
const payload = {
video_id: currentVideo.video_id,
annotations: {
responses: annotations,
axis_definitions: evaluationAxes.map(({ id, title, summary }) => ({
id,
title,
summary
})),
evaluation_items: currentVideo.evaluationItems,
temporal_constraints: currentVideo.constraints,
video_metadata: currentVideo.metadata,
source_details: currentVideo.source_details
},
user: annotator || "anonymous"
};
try {
await axios.post(`${API_BASE_URL}/save_annotation`, payload);
alert("Annotations saved.");
// advance to next video if available
if (currentIndex < videos.length - 1) {
setCurrentIndex((i) => i + 1);
// reset annotations for next video
setAnnotations(createEmptyAnnotations(videos[currentIndex + 1]));
} else {
alert("All videos completed.");
}
} catch (err) {
console.error(err);
const status = err.response?.status ? `Status: ${err.response.status}. ` : "";
const detail = err.response?.data?.detail || err.message || "Unknown error";
alert(`Error saving annotations.\n${status}${detail}\nBackend: ${API_BASE_URL}`);
}
};
if (!hasStarted) {
return (
Annotation task
T2AV Video Evaluation
The goal of this evaluation is to assess whether a
Text-to-Audio-Video generation model correctly follows the given
prompt by producing the required sounds, visual actions, and
temporal constraints. Your annotations should reflect only what is
actually visible and audible in the video, not what the prompt says
should happen.
Event Types
Speech: spoken words, such as "okay", "help", or "no".
Silent action: a soundless gesture, such as a thumbs-up, head nod, or hand wave.
Sounding action: an action that produces a sound, such as a clap, stomp, or thigh slap.
What You'll See for Each Video
The generation prompt for the video.
A target event checklist of the required speech, action, and sound events.
The temporal constraints, such as count, order, or synchronization.
A short description of each target action or sound.
How to Evaluate
Watch the full video before selecting labels.
Use slower playback when timing or synchronization is unclear.
Base every decision only on the provided video and audio.
Each axis already lists only the items relevant to that video, so judge exactly the items shown.
Labels
Yes: the specific listed item is clearly satisfied.
No: the specific listed item is clearly violated or missing. Treat ambiguous or hard-to-judge cases as No.
Temporal Count is not Yes/No: type the number of times the event actually occurs.
Labeling Order
Audio Content first: is the required speech or sound correct?
Then Visual Content: is the required visual action correct?
Then Audio-Visual Synchronization, only when the matching audio and visual items are correct enough to compare.
Then Temporal Relation and Temporal Count, only when the relevant audio, visual, and synchronization items are correct enough to identify the event.
Finally Constraint Following: are the non-core prompt constraints satisfied?
{evaluationAxes.map((axis) => (
{axis.title}
{axis.summary}
{axis.details.map((detail) => (
{detail}
))}
))}
Temporal Constraint Types
Count: the target behavior must occur exactly N times. Count only clear, completed occurrences.
Order: the required events must appear in the exact stated order, without reversal or unintended overlap.
Synchronization: the required events must occur simultaneously or with tight temporal alignment.
Rhythm: the behavior must repeat with the required rhythm.
Spacing: the gaps between events must match the spacing required in the prompt.
Absolute timing: onsets, pauses, and durations must match the exact times in the prompt.
Start/stop: the behavior must start and stop according to the trigger in the prompt.
Important Notes
Each axis lists only the items required by the prompt, so you never need to mark an item as "not applicable".
If the prompt specifies the right hand but the left hand is used, mark that visual item No. Other axes can still be Yes if otherwise satisfied.
If a required event does not appear at all, mark every item that depends on it No.
When an item is genuinely ambiguous due to video quality, occlusion, or camera angle, mark it No.
Rationales
Write a short reason (1-2 sentences) for every label.
Describe only observable evidence from the video; do not infer the model's intention.
Mention the specific sound, gesture, count, order, or timing you observed.
Do not leave rationale fields blank.
Examples: "The word 'help' is clearly audible once." or "The person raises a hand, but it is not a clear right-hand thumbs-up."
Example/Test Case
Watch the example video to calibrate your annotations before starting.