FastH3-Omni / composer.js
multimodalart's picture
multimodalart HF Staff
Start with an empty composer; dim placeholders
eb7e170 verified
Raw History Blame Contribute Delete
21.1 kB
const LIMITS = { image: 9, video: 3, audio: 3 };
const NAMES = { image: "Picture", video: "Video", audio: "Audio" };
const PLACEHOLDERS = {
image: "who or what is this? e.g. a woman with a curly afro in a white t-shirt",
video: "what to take from it? e.g. the dance moves and camera push",
audio: "what is it? e.g. her voice, or upbeat synth-pop",
};
const VIDEO_MIN = 2, VIDEO_MAX = 15;
const $ = (sel) => element.querySelector(sel);
const initial = props.value || {};
const state = {
mode: initial.mode || "builder",
refs: (initial.refs || []).map((ref) => ({ ...ref, id: ref.id || crypto.randomUUID() })),
builder: Object.assign({ scene: "", sound: "", music: "", style: "realistic photographic" }, initial.builder || {}),
write: initial.write || "",
writeEdited: !!initial.writeEdited,
};
let lastPushed = "";
let upsampling = false;
function kindOf(file) {
const type = file.type || "";
if (type.startsWith("image/")) return "image";
if (type.startsWith("video/")) return "video";
if (type.startsWith("audio/")) return "audio";
const ext = (file.name.split(".").pop() || "").toLowerCase();
if (["png", "jpg", "jpeg", "webp", "gif", "bmp"].includes(ext)) return "image";
if (["mp4", "mov", "webm", "mkv", "avi"].includes(ext)) return "video";
if (["wav", "mp3", "flac", "ogg", "m4a", "aac", "opus"].includes(ext)) return "audio";
return null;
}
function labelFor(index) {
const ref = state.refs[index];
let n = 0;
for (let i = 0; i <= index; i++) if (state.refs[i].kind === ref.kind) n++;
return `<${NAMES[ref.kind]} ${n}>`;
}
function labels() {
return state.refs.map((ref, index) => ({ ref, index, label: labelFor(index) }));
}
function subjects() {
return labels().filter(({ ref }) => ref.kind !== "audio");
}
function count(kind) {
return state.refs.filter((ref) => ref.kind === kind).length;
}
function escapeHtml(text) {
return String(text).replace(/[&<>"']/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[c]);
}
function clean(text) {
return (text || "").trim().replace(/[.\s]+$/, "");
}
function buildPrompt() {
const b = state.builder;
const all = labels();
const visual = subjects();
const audio = all.filter(({ ref }) => ref.kind === "audio");
const defs = [];
const retention = [];
visual.forEach(({ ref, label }, k) => {
const subject = `<Subject ${k + 1}>`;
const what = clean(ref.desc) || (ref.kind === "video" ? "the subject and motion of the reference video" : "the reference subject");
defs.push(`${subject} is ${what}, shown in ${label}.`);
retention.push(`${subject}: fully_preserved - the identity, appearance and details from ${label} are kept.`);
});
audio.forEach(({ ref, label }) => {
defs.push(`${label} is ${clean(ref.desc) || "the voice and sound reference"}.`);
});
const names = visual.map(({ ref }, k) => clean(ref.desc) || `<Subject ${k + 1}>`);
const summaryWho = names.length ? names.join(" and ") : "the scene";
let scene = (b.scene || "").trim();
if (scene && visual.length && !/<Subject \d+>/.test(scene) && /^[a-z]/.test(scene)) scene = `<Subject 1> ${scene}`;
const style = clean(b.style) || "realistic photographic";
let sound = clean(b.sound) || "natural ambient sound that matches the scene";
const unmentioned = audio.filter(({ label }) => !sound.includes(label)).map(({ label }) => label);
if (unmentioned.length) sound += `, following ${unmentioned.join(" and ")}`;
return [
`subject_definitions:\n${defs.join("\n") || "<Subject 1> is the subject of the video."}`,
`summary:\nA short video in a ${style} style of ${summaryWho}.`,
`retention_analysis:\n${retention.join("\n") || "none"}`,
`detailed_description:\nThe target video is in a ${style} style.\n[Shot 1] ${scene || "The subject moves naturally in the scene."}`,
`overall_soundscape:\n${sound}.`,
`non_diegetic_music:\n${clean(b.music) || "none"}.`,
].join("\n\n");
}
function upsampleSource() {
if (state.mode === "write") return state.write;
const parts = [];
subjects().forEach(({ ref, label }, k) => parts.push(`${label} shows ${clean(ref.desc) || `subject ${k + 1}`}.`));
labels().filter(({ ref }) => ref.kind === "audio").forEach(({ ref, label }) => parts.push(`${label} is ${clean(ref.desc) || "the voice and sound reference"}.`));
const b = state.builder;
let scene = (b.scene || "").replace(/<Subject (\d+)>/g, (_, n) => (subjects()[n - 1] ? clean(subjects()[n - 1].ref.desc) || `subject ${n}` : `subject ${n}`));
if (scene) parts.push(scene.trim());
if (clean(b.sound)) parts.push(`Sound: ${clean(b.sound)}.`);
if (clean(b.music) && clean(b.music).toLowerCase() !== "none") parts.push(`Music: ${clean(b.music)}.`);
if (clean(b.style) && clean(b.style) !== "realistic photographic") parts.push(`Style: ${clean(b.style)}.`);
return parts.join(" ");
}
function finalPrompt() {
return state.mode === "write" ? state.write : buildPrompt();
}
function push() {
const value = {
mode: state.mode,
refs: state.refs.map(({ id, kind, path, url, name, desc, duration, uploading, error }) => ({ id, kind, path, url, name, desc, duration, uploading: !!uploading, error: error || null })),
builder: { ...state.builder },
write: state.write,
writeEdited: state.writeEdited,
prompt: finalPrompt(),
upsample_source: upsampleSource(),
};
lastPushed = JSON.stringify(value);
props.value = value;
}
function insertAt(target, text) {
const start = target.selectionStart ?? target.value.length;
const end = target.selectionEnd ?? target.value.length;
const before = target.value.slice(0, start);
const pad = before && !/\s$/.test(before) ? " " : "";
target.value = before + pad + text + " " + target.value.slice(end);
const caret = start + pad.length + text.length + 1;
target.focus();
target.setSelectionRange(caret, caret);
target.dispatchEvent(new Event("input", { bubbles: true }));
}
function tagButton(label, kind, onClick, extra = "") {
const button = document.createElement("button");
button.type = "button";
button.className = `fh-tag ${kind === "audio" ? "is-audio" : kind === "video" ? "is-video" : kind === "subject" ? "is-subject" : ""} ${extra}`;
button.textContent = label;
button.title = "Insert into the prompt";
button.addEventListener("click", (event) => { event.preventDefault(); event.stopPropagation(); onClick(); });
return button;
}
function insertLabel(label) {
if (state.mode === "write") {
insertAt($("[data-write]"), label);
} else {
insertAt($('[data-b="scene"]'), label);
}
}
function fmtSeconds(seconds) {
return seconds == null ? "" : `${seconds.toFixed(1)}s`;
}
function renderTray() {
const tray = $("[data-tray]");
tray.querySelectorAll(".fh-card").forEach((card) => card.remove());
const drop = $("[data-drop]");
labels().forEach(({ ref, index, label }) => {
const card = document.createElement("div");
card.className = "fh-card" + (ref.error ? " fh-bad" : "");
card.draggable = true;
card.dataset.index = index;
const thumb = document.createElement("div");
thumb.className = "fh-thumb";
if (ref.kind === "image") {
thumb.innerHTML = `<img alt="" src="${escapeHtml(ref.preview || ref.url || "")}">`;
} else if (ref.kind === "video") {
thumb.innerHTML = `<video muted playsinline preload="metadata" src="${escapeHtml((ref.preview || ref.url || "") + "#t=0.1")}"></video>`;
const video = thumb.querySelector("video");
card.addEventListener("mouseenter", () => video.play().catch(() => {}));
card.addEventListener("mouseleave", () => { video.pause(); });
} else {
thumb.innerHTML = `<span class="fh-audio-glyph">♫</span>`;
card.addEventListener("click", (event) => {
if (event.target.closest("button")) return;
if (!ref._audio) ref._audio = new Audio(ref.preview || ref.url);
ref._audio.paused ? ref._audio.play() : ref._audio.pause();
});
card.title = "Click to play";
}
card.appendChild(thumb);
const foot = document.createElement("div");
foot.className = "fh-card-foot";
foot.appendChild(tagButton(label, ref.kind, () => insertLabel(label)));
const meta = document.createElement("span");
meta.className = "fh-meta";
meta.textContent = ref.error ? "!" : fmtSeconds(ref.duration);
foot.appendChild(meta);
card.appendChild(foot);
const remove = document.createElement("button");
remove.type = "button";
remove.className = "fh-x";
remove.textContent = "×";
remove.title = "Remove";
remove.addEventListener("click", (event) => { event.stopPropagation(); removeRef(index); });
card.appendChild(remove);
if (ref.uploading) {
const busy = document.createElement("div");
busy.className = "fh-busy";
busy.textContent = "uploading…";
card.appendChild(busy);
}
card.addEventListener("dragstart", (event) => {
card.classList.add("is-dragging");
event.dataTransfer.setData("text/fh-index", String(index));
event.dataTransfer.effectAllowed = "move";
});
card.addEventListener("dragend", () => card.classList.remove("is-dragging"));
card.addEventListener("dragover", (event) => {
if (!event.dataTransfer.types.includes("text/fh-index")) return;
event.preventDefault();
card.classList.add("is-target");
});
card.addEventListener("dragleave", () => card.classList.remove("is-target"));
card.addEventListener("drop", (event) => {
const from = event.dataTransfer.getData("text/fh-index");
card.classList.remove("is-target");
if (from === "") return;
event.preventDefault();
event.stopPropagation();
moveRef(Number(from), index);
});
tray.insertBefore(card, drop);
});
drop.hidden = state.refs.length >= LIMITS.image + LIMITS.video + LIMITS.audio;
$("[data-counts]").textContent = `${count("image")}/${LIMITS.image} images · ${count("video")}/${LIMITS.video} videos · ${count("audio")}/${LIMITS.audio} audio`;
const hint = $("[data-ref-hint]");
const bad = state.refs.find((ref) => ref.error);
if (bad) {
hint.textContent = bad.error;
hint.className = "fh-hint is-error";
} else if (!state.refs.length) {
hint.textContent = "Add at least one image or video. Each reference gets a tag you can use in the prompt.";
hint.className = "fh-hint";
} else if (!state.refs.some((ref) => ref.kind !== "audio")) {
hint.textContent = "Audio needs an image or a video alongside it.";
hint.className = "fh-hint is-error";
} else {
hint.textContent = "Drag to reorder: the order numbers the tags. Click a tag to insert it into the prompt.";
hint.className = "fh-hint";
}
}
function renderBuilder() {
const host = $("[data-refdescs]");
host.innerHTML = "";
if (!state.refs.length) {
host.innerHTML = `<div class="fh-empty">Add references above, then describe what each one is.</div>`;
}
let subjectIndex = 0;
labels().forEach(({ ref, label }) => {
const row = document.createElement("div");
row.className = "fh-refdesc";
const lead = document.createElement("div");
lead.className = "fh-refdesc-lead";
lead.appendChild(tagButton(label, ref.kind, () => insertLabel(label)));
if (ref.kind !== "audio") {
subjectIndex++;
const arrow = document.createElement("span");
arrow.className = "fh-arrow";
arrow.textContent = `→ <Subject ${subjectIndex}>`;
lead.appendChild(arrow);
}
const input = document.createElement("input");
input.type = "text";
input.placeholder = PLACEHOLDERS[ref.kind];
input.value = ref.desc || "";
input.addEventListener("input", () => { ref.desc = input.value; renderSubjectChips(); renderPreview(); push(); });
row.appendChild(lead);
row.appendChild(input);
host.appendChild(row);
});
renderSubjectChips();
renderPreview();
}
function renderSubjectChips() {
const bar = $("[data-subject-chips]");
bar.innerHTML = "";
const visual = subjects();
if (!visual.length) return;
const lead = document.createElement("span");
lead.className = "fh-chipbar-label";
lead.textContent = "Insert:";
bar.appendChild(lead);
visual.forEach(({ ref }, k) => {
const label = `<Subject ${k + 1}>`;
const chip = tagButton(label, "subject", () => insertAt($('[data-b="scene"]'), label));
if (clean(ref.desc)) chip.title = clean(ref.desc);
bar.appendChild(chip);
});
labels().filter(({ ref }) => ref.kind === "audio").forEach(({ label }) => {
bar.appendChild(tagButton(label, "audio", () => insertAt($('[data-b="sound"]'), label)));
});
}
function renderPreview() {
$("[data-preview]").textContent = buildPrompt();
}
function renderWrite() {
const bar = $("[data-write-chips]");
bar.innerHTML = "";
if (state.refs.length) {
const lead = document.createElement("span");
lead.className = "fh-chipbar-label";
lead.textContent = "Insert:";
bar.appendChild(lead);
}
const text = state.write || "";
labels().forEach(({ ref, label }) => {
bar.appendChild(tagButton(label, ref.kind, () => insertAt($("[data-write]"), label), text.includes(label) ? "" : "is-missing"));
});
const textarea = $("[data-write]");
if (textarea.value !== text) textarea.value = text;
renderLint();
}
function renderLint() {
const lint = $("[data-lint]");
const text = state.write || "";
const known = new Set(labels().map(({ label }) => label));
const used = [...new Set(text.match(/<(Picture|Video|Audio) \d+>/g) || [])];
const unknown = used.filter((label) => !known.has(label));
const missing = [...known].filter((label) => !text.includes(label));
const structured = /subject_definitions:/.test(text);
const parts = [];
if (text.includes("<removed>")) parts.push(`<span class="is-err">The prompt still refers to a removed reference (&lt;removed&gt;)</span>`);
if (!text.trim()) parts.push(`<span>Write a prompt, or switch to Builder.</span>`);
if (unknown.length) parts.push(`<span class="is-err">No reference for ${escapeHtml(unknown.join(", "))}</span>`);
if (missing.length && text.trim()) parts.push(`<span class="is-warn">Not mentioned: ${escapeHtml(missing.join(", "))}</span>`);
if (text.trim() && !unknown.length && !missing.length && !text.includes("<removed>")) parts.push(`<span class="is-ok">✓ every reference is mentioned</span>`);
if (text.trim()) parts.push(`<span>${structured ? "H3 six-section prompt, used as is" : "Plain prompt, wrapped into the H3 format on generate"}</span>`);
lint.innerHTML = parts.join("");
}
function renderMode() {
element.querySelectorAll("[data-mode]").forEach((button) => button.classList.toggle("is-active", button.dataset.mode === state.mode));
$('[data-pane="builder"]').hidden = state.mode !== "builder";
$('[data-pane="write"]').hidden = state.mode !== "write";
}
function renderBuilderFields() {
element.querySelectorAll("[data-b]").forEach((input) => {
const key = input.dataset.b;
if (input.value !== (state.builder[key] || "")) input.value = state.builder[key] || "";
});
}
function renderAll() {
renderMode();
renderTray();
renderBuilderFields();
renderBuilder();
renderWrite();
}
function relabelText(before) {
if (!state.write) return;
if (!state.writeEdited) {
state.write = buildPrompt();
return;
}
const after = labels();
const map = {};
before.forEach(({ id, label }) => {
const now = after.find(({ ref }) => ref.id === id);
map[label] = now ? now.label : null;
});
state.write = state.write.replace(/<(Picture|Video|Audio) \d+>/g, (label) => (label in map ? (map[label] === null ? "<removed>" : `\u0000${map[label]}\u0000`) : label)).replace(/\u0000/g, "");
}
function snapshotLabels() {
return labels().map(({ ref, label }) => ({ id: ref.id, label }));
}
function removeRef(index) {
const before = snapshotLabels();
const [ref] = state.refs.splice(index, 1);
if (ref && ref._audio) ref._audio.pause();
relabelText(before);
renderAll();
push();
}
function moveRef(from, to) {
if (from === to) return;
const before = snapshotLabels();
const [ref] = state.refs.splice(from, 1);
state.refs.splice(to, 0, ref);
relabelText(before);
renderAll();
push();
}
function mediaDuration(url, kind) {
return new Promise((resolve) => {
const media = document.createElement(kind === "video" ? "video" : "audio");
media.preload = "metadata";
media.onloadedmetadata = () => resolve(Number.isFinite(media.duration) ? media.duration : null);
media.onerror = () => resolve(null);
media.src = url;
});
}
async function addFiles(files) {
for (const file of files) {
const kind = kindOf(file);
if (!kind) continue;
if (count(kind) >= LIMITS[kind]) {
const hint = $("[data-ref-hint]");
hint.textContent = `At most ${LIMITS[kind]} ${kind} references.`;
hint.className = "fh-hint is-error";
continue;
}
const ref = { id: crypto.randomUUID(), kind, name: file.name, desc: "", preview: URL.createObjectURL(file), uploading: true };
state.refs.push(ref);
if (kind !== "image") {
ref.duration = await mediaDuration(ref.preview, kind);
if (kind === "video" && ref.duration != null && (ref.duration < VIDEO_MIN || ref.duration > VIDEO_MAX)) {
ref.error = `${file.name} is ${ref.duration.toFixed(1)} s; reference videos must be ${VIDEO_MIN}–${VIDEO_MAX} s.`;
}
}
renderAll();
push();
upload(file).then(({ path, url }) => {
ref.path = path;
ref.url = url;
ref.uploading = false;
renderTray();
push();
}).catch((error) => {
ref.uploading = false;
ref.error = `Upload failed for ${file.name}: ${error}`;
renderTray();
push();
});
}
}
const fileInput = $("[data-file]");
$("[data-drop]").addEventListener("click", () => fileInput.click());
fileInput.addEventListener("change", () => { addFiles([...fileInput.files]); fileInput.value = ""; });
const tray = $("[data-tray]");
tray.addEventListener("dragover", (event) => {
if (!event.dataTransfer.types.includes("Files")) return;
event.preventDefault();
tray.classList.add("is-over");
});
tray.addEventListener("dragleave", (event) => { if (!tray.contains(event.relatedTarget)) tray.classList.remove("is-over"); });
tray.addEventListener("drop", (event) => {
tray.classList.remove("is-over");
if (!event.dataTransfer.files.length) return;
event.preventDefault();
addFiles([...event.dataTransfer.files]);
});
element.querySelectorAll("[data-mode]").forEach((button) => button.addEventListener("click", () => {
if (button.dataset.mode === state.mode) return;
if (button.dataset.mode === "write" && (!state.write.trim() || !state.writeEdited)) {
state.write = buildPrompt();
state.writeEdited = false;
}
state.mode = button.dataset.mode;
renderAll();
push();
}));
$("[data-to-write]").addEventListener("click", (event) => {
event.preventDefault();
state.write = buildPrompt();
state.writeEdited = false;
state.mode = "write";
renderAll();
push();
});
element.querySelectorAll("[data-b]").forEach((input) => input.addEventListener("input", () => {
state.builder[input.dataset.b] = input.value;
renderPreview();
push();
}));
$("[data-write]").addEventListener("input", (event) => {
state.write = event.target.value;
state.writeEdited = true;
renderWrite();
push();
});
const upsampleButton = $("[data-upsample]");
function setUpsampling(busy) {
upsampling = busy;
upsampleButton.disabled = busy;
upsampleButton.classList.toggle("is-busy", busy);
$("[data-upsample-label]").textContent = busy ? "Upsampling…" : "Upsample";
}
upsampleButton.addEventListener("click", () => {
if (upsampling) return;
setUpsampling(true);
push();
trigger("edit");
});
function loadExample(example) {
state.refs = example.refs.map((ref) => ({ ...ref, id: crypto.randomUUID(), preview: ref.url }));
state.builder = Object.assign({ scene: "", sound: "", music: "", style: "realistic photographic" }, example.builder);
state.write = "";
state.writeEdited = false;
state.mode = "builder";
renderAll();
push();
}
const examples = (props.value && props.value.examples) || [];
const exampleRow = $("[data-examples]");
examples.forEach((example) => {
const button = document.createElement("button");
button.type = "button";
button.className = "fh-example";
button.innerHTML = `<img alt="" src="${escapeHtml(example.refs[0].url)}"><span>${escapeHtml(example.title)}</span>`;
button.addEventListener("click", () => loadExample(example));
exampleRow.appendChild(button);
});
if (!examples.length) $(".fh-examples").hidden = true;
watch("value", () => {
const incoming = props.value || {};
if (JSON.stringify(incoming) === lastPushed) return;
if ("upsampled" in incoming) {
setUpsampling(false);
if (incoming.upsampled) {
state.write = incoming.upsampled;
state.writeEdited = true;
state.mode = "write";
}
renderAll();
push();
}
});
renderAll();