Architecture
Transformer encoder mapping multimodal features to cortical surface activity
Encoders
V-JEPA2 (video) · Wav2Vec-BERT 2.0 (audio) · LLaMA 3.2-3B (text)
Preprocessing
WhisperX extracts word-level timestamps from audio/video, enabling the text encoder to process speech with precise timing
Output
Predicted fMRI BOLD responses on the fsaverage5 cortical mesh — 20,484 vertices, 1 TR = 1 s
Training data
700+ healthy subjects exposed to images, podcasts, videos, and text (naturalistic paradigm)
License
CC BY-NC 4.0 — research and non-commercial use only
"""
NOTES_HTML = """
"""
# ── Singletons ─────────────────────────────────────────────────────────────────
_model = None
_plotter = None
_mesh_cache = None
def _load_model():
global _model, _plotter
if _model is None:
from tribev2.demo_utils import TribeModel
from tribev2.plotting import PlotBrain
hf_token = os.environ.get("HF_TOKEN")
if hf_token:
from huggingface_hub import login
login(token=hf_token, add_to_git_credential=False)
_model = TribeModel.from_pretrained("facebook/tribev2", cache_folder=CACHE_FOLDER)
_plotter = PlotBrain(mesh="fsaverage5")
return _model, _plotter
def _load_mesh():
global _mesh_cache
if _mesh_cache is None:
from nilearn import datasets, surface
fsaverage = datasets.fetch_surf_fsaverage("fsaverage5")
coords_L, faces_L = surface.load_surf_mesh(fsaverage.pial_left)
coords_R, faces_R = surface.load_surf_mesh(fsaverage.pial_right)
_mesh_cache = (
np.array(coords_L), np.array(faces_L),
np.array(coords_R), np.array(faces_R),
)
return _mesh_cache
# ── 3-D brain builder ──────────────────────────────────────────────────────────
def build_3d_figure(preds: np.ndarray, vmin_val: float = 0.5) -> str:
"""Return an HTML iframe with interactive 3-D brain — white base,
fire activation overlay, centered slider."""
import plotly.graph_objects as go
import json
import html as _html
coords_L, faces_L, coords_R, faces_R = _load_mesh()
n_verts_L = coords_L.shape[0]
n_t = preds.shape[0]
# Normalization: same threshold as the timeline slider
vmax = np.percentile(preds, 99)
vmin = vmin_val
BG = "#1a1a2e"
MONO = "ui-monospace, 'Cascadia Code', 'Source Code Pro', monospace"
# White base colorscale: 0→white, fire only above threshold
WHITE_FIRE = [
[0.00, "rgb(245,245,245)"],
[0.25, "rgb(220,180,160)"],
[0.45, "rgb(200,60,10)"],
[0.65, "rgb(240,120,0)"],
[0.80, "rgb(255,200,20)"],
[1.00, "rgb(255,255,220)"],
]
mesh_kw = dict(
colorscale=WHITE_FIRE, cmin=0, cmax=1, showscale=False,
flatshading=False, hoverinfo="skip",
lighting=dict(ambient=0.60, diffuse=0.85, specular=0.25, roughness=0.45),
lightposition=dict(x=80, y=180, z=200),
)
def _vals(t):
v = preds[t]
return np.clip((v - vmin) / max(vmax - vmin, 1e-8), 0, 1)
def _traces(t):
vn = _vals(t)
offset = 8.0
tL = go.Mesh3d(
x=coords_L[:, 0] - offset, y=coords_L[:, 1], z=coords_L[:, 2],
i=faces_L[:, 0], j=faces_L[:, 1], k=faces_L[:, 2],
intensity=vn[:n_verts_L], name="Left", **mesh_kw)
tR = go.Mesh3d(
x=coords_R[:, 0] + offset, y=coords_R[:, 1], z=coords_R[:, 2],
i=faces_R[:, 0], j=faces_R[:, 1], k=faces_R[:, 2],
intensity=vn[n_verts_L:], name="Right", **mesh_kw)
return tL, tR
def _intensity_only(t):
vn = _vals(t)
return [go.Mesh3d(intensity=vn[:n_verts_L]),
go.Mesh3d(intensity=vn[n_verts_L:])]
tL0, tR0 = _traces(0)
frames = [
go.Frame(data=_intensity_only(t), name=str(t),
layout=go.Layout(title_text=f"t = {t} s"))
for t in range(n_t)
]
slider_steps = [
dict(args=[[str(t)], dict(frame=dict(duration=0, redraw=True),
mode="immediate", transition=dict(duration=0))],
label=str(t), method="animate")
for t in range(n_t)
]
fig = go.Figure(
data=[tL0, tR0],
frames=frames,
layout=go.Layout(
height=500,
paper_bgcolor=BG,
plot_bgcolor=BG,
scene=dict(
bgcolor=BG,
xaxis=dict(visible=False),
yaxis=dict(visible=False),
zaxis=dict(visible=False),
camera=dict(
eye=dict(x=0, y=-1.9, z=0.4),
up=dict(x=0, y=0, z=1),
),
aspectmode="data",
),
margin=dict(l=0, r=0, t=8, b=70),
title=dict(
text="t = 0 s — drag to rotate · scroll to zoom",
font=dict(color="#9ca3af", family=MONO, size=11),
x=0.5,
),
updatemenus=[],
sliders=[dict(
active=0, steps=slider_steps,
currentvalue=dict(
prefix="t = ", suffix=" s",
font=dict(color="#9ca3af", family=MONO, size=11),
visible=True, xanchor="center",
),
pad=dict(b=8, t=8),
len=0.85, x=0.5, xanchor="center", y=0,
bgcolor="#111827", bordercolor="#1f2937",
tickcolor="#374151",
font=dict(color="#6b7280", family=MONO, size=10),
)],
),
)
inner_html = fig.to_html(
include_plotlyjs=True,
full_html=True,
config={"responsive": True, "displayModeBar": False},
)
srcdoc = _html.escape(inner_html, quote=True)
return (
f'