t2av_eval / frontend /src /lib /promptSchema.js
Youngsun's picture
T2AV eval site: guideline to training to eval, dghadiya storage
44e5f85
Raw
History Blame Contribute Delete
5.12 kB
// Normalizes raw entries from data/prompts.json (relative_50_v3_prompts_newid.json)
// into the shape the UI works with. See T2AV_guideline_final_updated.html for the
// axis vocabulary (A / V / AVB / TC_REL / TC_COUNT / CC) this is built around.
const MODALITY_LABELS = {
speech: "Speech",
silent_action: "Silent action",
sounding_action: "Sounding action"
};
// Which of the per-event axes apply to a given event modality.
// Silent actions have no audio component, so no A and no AVB (per guideline s2/s4).
function axesForModality(modality) {
if (modality === "silent_action") {
return { a: false, v: true, avb: false };
}
return { a: true, v: true, avb: true };
}
function describeEvent(event) {
if (event.modality === "speech") {
return `Speech: say "${event.text}"`;
}
const sideAndPart = [event.body_side, event.body_part].filter(Boolean).join(" ");
if (event.modality === "silent_action") {
const detail = sideAndPart ? ` (${sideAndPart})` : "";
return `Silent action: ${event.action}${detail}`;
}
// sounding_action
const actingSide = [event.acting_body_side, event.acting_body_part].filter(Boolean).join(" ");
const targetSide = [event.target_body_side, event.target_body_part].filter(Boolean).join(" ");
const detail = [actingSide && `with ${actingSide}`, targetSide && `on ${targetSide}`]
.filter(Boolean)
.join(" ");
return `Sounding action: ${event.action}${detail ? ` (${detail})` : ""}`;
}
function normalizeEvent(event) {
return {
id: event.id,
modality: event.modality,
modalityLabel: MODALITY_LABELS[event.modality] || event.modality,
actor: event.actor,
label: describeEvent(event),
definition: event.definition,
axes: axesForModality(event.modality)
};
}
// Per-event expected count, derived from the temporal_constraint's value shape.
// Defaults to 1 for any event not explicitly given a count (relation types
// without counting — synchronization/order — always expect exactly one).
function expectedCountsFromConstraint(tc, eventIds) {
const counts = Object.fromEntries(eventIds.map((id) => [id, 1]));
if (!tc || !tc.value) return counts;
if (tc.relation_type === "sync_count" && Array.isArray(tc.value.events)) {
tc.value.events.forEach((id) => {
counts[id] = tc.value.count;
});
} else if (tc.relation_type === "order_count" && tc.value.event_counts) {
Object.entries(tc.value.event_counts).forEach(([id, count]) => {
counts[id] = count;
});
}
return counts;
}
function describeTemporalConstraint(tc, eventsById) {
if (!tc || !tc.value) return "No temporal constraint specified.";
const label = (id) => (eventsById[id] ? eventsById[id].label : id);
switch (tc.relation_type) {
case "synchronization":
return `Synchronization: ${tc.value.events.map(label).join(" + ")} must all occur at the same moment.`;
case "sync_count":
return `Synchronization x${tc.value.count}: ${tc.value.events
.map(label)
.join(" + ")} must occur together, repeated exactly ${tc.value.count} times.`;
case "order":
return `Order: ${tc.value.sequence.map(label).join(" -> then -> ")}.`;
case "order_count": {
const parts = tc.value.ordered_events.map(
(id) => `${label(id)} x${tc.value.event_counts[id] ?? 1}`
);
return `Order: ${parts.join(" -> then -> ")}.`;
}
default:
return `${tc.relation_type}: ${JSON.stringify(tc.value)}`;
}
}
export function normalizePromptItem(raw) {
const events = raw.events.map(normalizeEvent);
const eventsById = Object.fromEntries(events.map((e) => [e.id, e]));
const expectedCounts = expectedCountsFromConstraint(raw.temporal_constraint, events.map((e) => e.id));
return {
itemId: raw.item_id,
canonicalId: raw.canonical_item_id,
modalityCombination: raw.modality_combination,
prompt: raw.generated_prompt,
sceneHint: raw.scene_hint,
cameraHint: raw.camera_hint,
events,
temporalConstraint: {
relationType: raw.temporal_constraint.relation_type,
timingType: raw.temporal_constraint.timing_type,
value: raw.temporal_constraint.value,
expectedCounts,
description: describeTemporalConstraint(raw.temporal_constraint, eventsById)
}
};
}
export function normalizePromptList(rawList) {
return rawList.map(normalizePromptItem);
}
// Creates an empty annotations object matching a normalized item's applicable
// axes, so the UI never has to special-case "does this event have an A field".
export function createEmptyAnnotations(item) {
const events = Object.fromEntries(
item.events.map((event) => {
const fields = {};
if (event.axes.a) fields.a = { label: "", rationale: "" };
if (event.axes.v) fields.v = { label: "", rationale: "" };
if (event.axes.avb) fields.avb = { label: "", rationale: "" };
fields.tcCount = { value: "", rationale: "" };
return [event.id, fields];
})
);
return {
events,
video: {
tcRel: { label: "", rationale: "" },
cc: { label: "", rationale: "" }
}
};
}