"""Ask-the-video tab for prerendered HF Space demo."""
from __future__ import annotations
import html
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
import streamlit as st
TimelineRows = list[dict[str, Any]]
@dataclass(frozen=True)
class CannedAnswer:
narrative: str
timeline: TimelineRows
crops: TimelineRows
@dataclass(frozen=True)
class CannedQuery:
prompt: str
aliases: list[str]
responder: Callable[[TimelineRows], CannedAnswer]
def _normalized(value: str) -> str:
return " ".join(value.lower().strip().split())
def _format_ts(timestamp: float) -> str:
minutes = int(timestamp // 60)
seconds = int(timestamp % 60)
return f"{minutes:02d}:{seconds:02d}"
def _track_rows(timeline_rows: TimelineRows) -> dict[int, TimelineRows]:
grouped: dict[int, TimelineRows] = {}
for row in timeline_rows:
track_id = int(row.get("track_id", -1))
if track_id < 0:
continue
grouped.setdefault(track_id, []).append(row)
return grouped
def _crops_from_rows(rows: TimelineRows, limit: int = 6) -> TimelineRows:
seen: set[tuple[int, int]] = set()
selected: TimelineRows = []
for row in rows:
key = (int(row.get("track_id", -1)), int(row.get("frame_idx", -1)))
if key in seen:
continue
seen.add(key)
selected.append(row)
if len(selected) >= limit:
break
return selected
def _answer_people_count(timeline_rows: TimelineRows) -> CannedAnswer:
tracks = sorted(_track_rows(timeline_rows).keys())
narrative = f"Detected {len(tracks)} tracked worker(s) in this pre-rendered shift: {', '.join(f'#{t}' for t in tracks)}."
return CannedAnswer(narrative=narrative, timeline=timeline_rows[:15], crops=_crops_from_rows(timeline_rows))
def _answer_longest_presence(timeline_rows: TimelineRows) -> CannedAnswer:
grouped = _track_rows(timeline_rows)
if not grouped:
return CannedAnswer("No tracks available for this video.", [], [])
best_track = -1
best_duration = -1.0
for track_id, rows in grouped.items():
timestamps = [float(row.get("timestamp_sec", 0.0)) for row in rows]
duration = max(timestamps) - min(timestamps)
if duration > best_duration:
best_track = track_id
best_duration = duration
winner_rows = grouped[best_track]
start_ts = _format_ts(float(winner_rows[0].get("timestamp_sec", 0.0)))
end_ts = _format_ts(float(winner_rows[-1].get("timestamp_sec", 0.0)))
narrative = (
f"Track {best_track} has the longest observed presence, roughly {best_duration:.1f}s "
f"(from {start_ts} to {end_ts})."
)
return CannedAnswer(narrative=narrative, timeline=winner_rows[:20], crops=_crops_from_rows(winner_rows))
def _answer_track_one(timeline_rows: TimelineRows) -> CannedAnswer:
grouped = _track_rows(timeline_rows)
rows = grouped.get(1) or []
if not rows:
return CannedAnswer("Track 1 is not present in this video.", [], [])
first = rows[0]
summary = first.get("narrative_summary") or "No narrative summary available."
narrative = f"Track 1 summary: {summary}"
return CannedAnswer(narrative=narrative, timeline=rows[:20], crops=_crops_from_rows(rows))
def _answer_yellow_worker(timeline_rows: TimelineRows) -> CannedAnswer:
rows = [row for row in timeline_rows if row.get("color_tag") == "yellow_top"]
if not rows:
return CannedAnswer("No worker tagged `yellow_top` was found in this sample.", [], [])
grouped = _track_rows(rows)
track_id = sorted(grouped.keys())[0]
summary = grouped[track_id][0].get("narrative_summary") or "No narrative summary available."
narrative = f"Yellow-top worker maps to track {track_id}. {summary}"
return CannedAnswer(narrative=narrative, timeline=grouped[track_id][:20], crops=_crops_from_rows(grouped[track_id]))
def _answer_anomalies(timeline_rows: TimelineRows) -> CannedAnswer:
anomalous = [
row
for row in timeline_rows
if isinstance(row.get("activity"), dict) and bool(row["activity"].get("anomaly"))
]
if not anomalous:
return CannedAnswer(
"No explicit safety anomaly flags were found in this pre-rendered timeline.",
timeline_rows[:12],
_crops_from_rows(timeline_rows),
)
narrative = f"Found {len(anomalous)} anomaly-tagged event(s)."
return CannedAnswer(narrative=narrative, timeline=anomalous[:30], crops=_crops_from_rows(anomalous))
def _answer_activities(timeline_rows: TimelineRows) -> CannedAnswer:
counts: dict[str, int] = {}
for row in timeline_rows:
activity = row.get("activity")
if isinstance(activity, dict):
label = str(activity.get("activity") or "unknown")
else:
label = "unknown"
counts[label] = counts.get(label, 0) + 1
if not counts:
return CannedAnswer("No activity labels available.", [], [])
ordered = sorted(counts.items(), key=lambda item: item[1], reverse=True)
narrative = "Observed activities: " + ", ".join(f"{name} ({count})" for name, count in ordered[:8]) + "."
return CannedAnswer(narrative=narrative, timeline=timeline_rows[:20], crops=_crops_from_rows(timeline_rows))
def _answer_busiest_moment(timeline_rows: TimelineRows) -> CannedAnswer:
moment_counts: dict[float, set[int]] = {}
for row in timeline_rows:
ts = float(row.get("timestamp_sec", 0.0))
track_id = int(row.get("track_id", -1))
moment_counts.setdefault(ts, set()).add(track_id)
if not moment_counts:
return CannedAnswer("No timestamp data available.", [], [])
peak_ts, tracks = max(moment_counts.items(), key=lambda item: len(item[1]))
peak_rows = [row for row in timeline_rows if float(row.get("timestamp_sec", 0.0)) == peak_ts]
narrative = (
f"Busiest timestamp is {_format_ts(peak_ts)} with {len(tracks)} concurrent tracked worker(s): "
f"{', '.join(f'#{track}' for track in sorted(tracks))}."
)
return CannedAnswer(narrative=narrative, timeline=peak_rows, crops=_crops_from_rows(peak_rows))
def _answer_all_tracks(timeline_rows: TimelineRows) -> CannedAnswer:
grouped = _track_rows(timeline_rows)
if not grouped:
return CannedAnswer("No tracks available in this timeline.", [], [])
descriptions: list[str] = []
selected: TimelineRows = []
for track_id in sorted(grouped.keys()):
rows = grouped[track_id]
first_ts = float(rows[0].get("timestamp_sec", 0.0))
last_ts = float(rows[-1].get("timestamp_sec", 0.0))
descriptions.append(f"#{track_id} ({_format_ts(first_ts)}-{_format_ts(last_ts)})")
selected.extend(rows[:1])
narrative = f"Tracked workers in this video: {', '.join(descriptions)}."
return CannedAnswer(narrative=narrative, timeline=selected, crops=_crops_from_rows(timeline_rows, limit=8))
CANNED_QUERIES: list[CannedQuery] = [
CannedQuery(
prompt="How many people worked this shift?",
aliases=["how many people", "people worked", "worker count", "number of workers"],
responder=_answer_people_count,
),
CannedQuery(
prompt="Who stayed the longest in view?",
aliases=["stayed the longest", "longest activity", "longest time"],
responder=_answer_longest_presence,
),
CannedQuery(
prompt="Show me track 1",
aliases=["track 1", "show track one"],
responder=_answer_track_one,
),
CannedQuery(
prompt="What did the yellow-top worker do?",
aliases=["yellow-top", "yellow top", "orange vest"],
responder=_answer_yellow_worker,
),
CannedQuery(
prompt="Are there any safety anomalies?",
aliases=["safety anomalies", "anomaly", "unsafe"],
responder=_answer_anomalies,
),
CannedQuery(
prompt="What activities happened?",
aliases=["activities happened", "what activities", "activity breakdown"],
responder=_answer_activities,
),
CannedQuery(
prompt="When was the warehouse busiest?",
aliases=["warehouse busiest", "busiest", "most people at once"],
responder=_answer_busiest_moment,
),
CannedQuery(
prompt="Show me all tracks",
aliases=["all tracks", "list tracks", "show tracks"],
responder=_answer_all_tracks,
),
]
def _match_query(prompt: str) -> CannedQuery | None:
normalized = _normalized(prompt)
for item in CANNED_QUERIES:
for alias in item.aliases:
if _normalized(alias) in normalized:
return item
return None
def _resolve_crop_path(crop_path: str | None, workspace_root: Path) -> Path | None:
if not crop_path:
return None
path = Path(crop_path)
if path.exists():
return path
fallback = workspace_root / crop_path
if fallback.exists():
return fallback
return None
def _render_timeline_expander(timeline_rows: TimelineRows, key_prefix: str) -> None:
if not timeline_rows:
return
with st.expander("Timeline details", expanded=False):
for index, row in enumerate(timeline_rows[:40]):
ts = _format_ts(float(row.get("timestamp_sec", 0.0)))
track_id = row.get("track_id", "n/a")
activity = row.get("activity")
label = activity.get("activity", "unknown") if isinstance(activity, dict) else "unknown"
st.markdown(f"`{ts}` • Track `{track_id}` • Activity `{label}`")
st.button(
f"Jump to {ts}",
key=f"{key_prefix}-jump-{index}",
type="tertiary",
disabled=True,
)
def _render_crops(candidates: TimelineRows, workspace_root: Path, key_prefix: str) -> None:
if not candidates:
return
st.markdown("**Highlighted crops**")
cols = st.columns(min(4, len(candidates)))
for index, row in enumerate(candidates[:8]):
crop = _resolve_crop_path(row.get("crop_path"), workspace_root)
caption = f"Track {row.get('track_id', '?')}"
with cols[index % len(cols)]:
if crop is not None:
st.image(str(crop), caption=caption, use_container_width=True)
else:
st.caption(f"{caption}: crop unavailable")
st.button(
f"Select {caption}",
key=f"{key_prefix}-select-{index}",
type="secondary",
disabled=True,
)
def _next_message_id(role: str) -> str:
counter = int(st.session_state.space_message_counter)
st.session_state.space_message_counter = counter + 1
return f"{role}-{counter}"
def _render_suggestion_chips() -> None:
st.markdown("#### Suggested Questions")
quick = CANNED_QUERIES[:2]
quick_cols = st.columns(2)
for idx, query in enumerate(quick):
if quick_cols[idx].button(query.prompt, key=f"space-suggest-quick-{idx}"):
st.session_state.space_pending_prompt = query.prompt
with st.expander("More suggestions", expanded=False):
extra = CANNED_QUERIES[2:]
if not extra:
st.caption("No more suggestions available.")
return
cols = st.columns(2)
for idx, query in enumerate(extra):
if cols[idx % 2].button(query.prompt, key=f"space-suggest-extra-{idx}"):
st.session_state.space_pending_prompt = query.prompt
def _render_messages_only(history: list[dict[str, Any]]) -> None:
if not history:
st.caption("Start the conversation with one of the suggested prompts.")
return
rows: list[str] = []
for message in history:
role = str(message.get("role", "assistant")).lower()
safe_text = html.escape(str(message.get("content", ""))).replace("\n", "
")
if role == "user":
rows.append(
f'