"""Predict cortical responses to a stimulus with TRIBE v2 (facebook/tribev2, CC BY-NC 4.0). A perception runs `encode_stimulus` on the model's inference endpoint (scripts/brain/tribe/handler.py, docs/harnesses.md § Perception). The command line is the same computation, for studies run offline: ~/tribe-env/bin/python scripts/brain/encode.py --text page.txt --out run/ ~/tribe-env/bin/python scripts/brain/encode.py --video clip.mp4 --out run/ ~/tribe-env/bin/python scripts/brain/encode.py --screen frame.png --seconds 20 --out run/ A text is spoken (text-to-speech) and transcribed back to timed words, as TRIBE's own demo does; a screen is a still held for `--seconds` as a silent video; a video or audio file is taken as it is. The model answers one prediction a second (its TR) on the fsaverage5 cortical mesh (20,484 vertices, left then right), for the average subject, in z-units of BOLD, shifted 5 s into the past for the haemodynamic lag. Writes into --out: preds.npz preds (T, 20484) float16 and times (T,) seconds rois.json every Destrieux parcel's time-mean and peak, per hemisphere, with its name features.npz the backbone features the model was given (time-mean per modality), for matched-control analyses: the same stimulus without the brain projection map.png the time-mean prediction on the inflated surface, four views summary.json what was run (model, checkpoint digest, stimulus, seconds, segments, duration), the hemisphere means, and the ten strongest parcels It fails plainly without the model, the GPU or the stimulus; it never writes a result it did not compute. """ import argparse import hashlib import json import os import pathlib import subprocess import time MODEL_ID = "facebook/tribev2" MESH = "fsaverage5" N_VERTICES = 20484 def sha256(path): h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def ffmpeg(): import imageio_ffmpeg return imageio_ffmpeg.get_ffmpeg_exe() def screen_video(png, seconds, out): """A still image held for `seconds`, with a silent audio track so the audio extractor has one.""" subprocess.run([ffmpeg(), "-y", "-loglevel", "error", "-loop", "1", "-framerate", "4", "-i", str(png), "-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", str(seconds), "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p", "-c:v", "libx264", "-preset", "fast", "-crf", "20", "-c:a", "aac", "-shortest", str(out)], check=True) return out def transcribe_on_cpu(): """TRIBE transcribes with whisperx under `uvx` on CUDA in float16; CTranslate2's aarch64 wheel has no CUDA, so here whisperx runs on the CPU in int8 (BRAIN_WHISPER_DEVICE=cuda keeps TRIBE's own).""" if os.environ.get("BRAIN_WHISPER_DEVICE", "cpu") == "cuda": return import json as _json import subprocess as _sp import tempfile from pathlib import Path import pandas as pd from tribev2 import eventstransforms as et codes = dict(english="en", french="fr", spanish="es", dutch="nl", chinese="zh") def get_transcript(wav_filename, language): if language not in codes: raise ValueError(f"Language {language} not supported") with tempfile.TemporaryDirectory() as out: cmd = ["uvx", "whisperx", str(wav_filename), "--model", "large-v3", "--language", codes[language], "--device", "cpu", "--compute_type", "int8", "--batch_size", "16", "--output_dir", out, "--output_format", "json"] if language == "english": cmd += ["--align_model", "WAV2VEC2_ASR_LARGE_LV60K_960H"] env = {k: v for k, v in os.environ.items() if k != "MPLBACKEND"} r = _sp.run(cmd, capture_output=True, text=True, env=env) if r.returncode != 0: raise RuntimeError(f"whisperx failed:\n{r.stderr[-2000:]}") transcript = _json.loads((Path(out) / f"{Path(wav_filename).stem}.json").read_text()) words = [] for i, seg in enumerate(transcript["segments"]): sentence = seg["text"].replace('"', "") for w in seg["words"]: if "start" in w: words.append({"text": w["word"].replace('"', ""), "start": w["start"], "duration": w["end"] - w["start"], "sequence_id": i, "sentence": sentence}) return pd.DataFrame(words) et.ExtractWordsFromAudio._get_transcript_from_audio = staticmethod(get_transcript) def load_model(cache): from tribev2.demo_utils import TribeModel transcribe_on_cpu() return TribeModel.from_pretrained(MODEL_ID, cache_folder=str(cache)) def checkpoint_digest(): from huggingface_hub import hf_hub_download return sha256(hf_hub_download(MODEL_ID, "best.ckpt")) class FeatureTap: """Catches the per-modality backbone features each batch carries into the model.""" def __init__(self, model): self.sums, self.counts = {}, {} self.handle = model.register_forward_pre_hook(self.hook, with_kwargs=False) def hook(self, module, args): import torch batch = args[0] for modality in ("text", "audio", "video"): if modality not in batch.data: continue x = batch.data[modality].to(torch.float32) # B, L, D, T (or B, D, T) if x.ndim == 3: x = x.unsqueeze(1) x = x.mean(dim=1) # mean over layers: B, D, T self.sums[modality] = self.sums.get(modality, 0) + x.sum(dim=(0, 2)).cpu().numpy() self.counts[modality] = self.counts.get(modality, 0) + x.shape[0] * x.shape[2] def means(self): return {m: self.sums[m] / max(self.counts[m], 1) for m in self.sums} def atlas(): """Destrieux parcels on fsaverage5: (labels_left, labels_right, names).""" from nilearn import datasets a = datasets.fetch_atlas_surf_destrieux() names = [n.decode() if isinstance(n, bytes) else str(n) for n in a["labels"]] return a["map_left"], a["map_right"], names def parcels(mean, peak, labels_l, labels_r, names): import numpy as np out = [] half = N_VERTICES // 2 for hemi, labels, lo in (("L", labels_l, 0), ("R", labels_r, half)): for idx, name in enumerate(names): if name in ("Unknown", "Medial_wall"): continue sel = np.where(labels == idx)[0] if not len(sel): continue out.append({"id": f"{hemi}_{name}", "hemisphere": hemi, "name": name, "vertices": int(len(sel)), "mean": float(mean[lo + sel].mean()), "peak": float(peak[lo + sel].mean())}) return out def plot_map(mean, out): import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np from nilearn import datasets, plotting fs = datasets.fetch_surf_fsaverage(MESH) half = N_VERTICES // 2 v = float(np.percentile(np.abs(mean), 99)) or 1.0 fig, axes = plt.subplots(1, 4, figsize=(16, 4), subplot_kw={"projection": "3d"}) views = [("left", "lateral", mean[:half]), ("left", "medial", mean[:half]), ("right", "lateral", mean[half:]), ("right", "medial", mean[half:])] for ax, (hemi, view, data) in zip(axes, views): plotting.plot_surf_stat_map(fs[f"infl_{hemi}"], data, hemi=hemi, view=view, bg_map=fs[f"sulc_{hemi}"], cmap="RdBu_r", vmax=v, symmetric_cbar=True, colorbar=(view == "medial" and hemi == "right"), axes=ax, threshold=v * 0.1) ax.set_title(f"{hemi} {view}", fontsize=10) fig.savefig(out, dpi=110, bbox_inches="tight") plt.close(fig) def encode_stimulus(model, kind, path, out, seconds=20.0, draw_map=True): """Predict the response to one stimulus with a loaded model, and write the files above into `out`. `kind` is text, video, audio or screen; `path` the file. → the summary.""" import numpy as np path = pathlib.Path(path) out = pathlib.Path(out) out.mkdir(parents=True, exist_ok=True) t0 = time.time() stimulus = {"kind": kind, "source": str(path), "sha256": sha256(path)} if kind == "screen": clip = out / "stimulus.mp4" screen_video(path, seconds, clip) stimulus["seconds"] = seconds events_arg = {"video_path": str(clip)} elif kind == "video": events_arg = {"video_path": str(path.resolve())} elif kind == "audio": events_arg = {"audio_path": str(path.resolve())} elif kind == "text": stimulus["words"] = len(path.read_text(encoding="utf-8").split()) events_arg = {"text_path": str(path.resolve())} else: raise ValueError(f"no stimulus of the kind {kind}") tap = FeatureTap(model._model) events = model.get_events_dataframe(**events_arg) preds, segments = model.predict(events, verbose=False) tap.handle.remove() if preds.shape[1] != N_VERTICES: raise RuntimeError(f"expected {N_VERTICES} vertices, got {preds.shape[1]}") times = np.array([float(s.start) for s in segments]) order = np.argsort(times) preds, times = preds[order], times[order] mean = preds.mean(axis=0) peak = preds.max(axis=0) np.savez_compressed(out / "preds.npz", preds=preds.astype(np.float16), times=times.astype(np.float32)) features = tap.means() np.savez_compressed(out / "features.npz", **{k: v.astype(np.float32) for k, v in features.items()}) labels_l, labels_r, names = atlas() rows = parcels(mean, peak, labels_l, labels_r, names) (out / "rois.json").write_text(json.dumps({"atlas": "Destrieux 2009 on fsaverage5 (nilearn)", "parcels": rows}, indent=1)) if draw_map: plot_map(mean, out / "map.png") half = N_VERTICES // 2 top = sorted(rows, key=lambda r: -r["mean"])[:10] summary = { "model": MODEL_ID, "checkpoint_sha256": checkpoint_digest(), "mesh": MESH, "vertices": N_VERTICES, "stimulus": stimulus, "segments": int(preds.shape[0]), "seconds_predicted": float(times[-1] - times[0] + 1) if len(times) else 0.0, "modalities": sorted(features), "feature_dims": {k: int(v.shape[0]) for k, v in features.items()}, "hemisphere_mean": {"left": float(mean[:half].mean()), "right": float(mean[half:].mean())}, "global_mean": float(mean.mean()), "global_peak": float(peak.max()), "top_parcels": [{"id": r["id"], "mean": r["mean"]} for r in top], "runtime_s": round(time.time() - t0, 1), "files": ["preds.npz", "rois.json", "features.npz"] + (["map.png"] if draw_map else []), } (out / "summary.json").write_text(json.dumps(summary, indent=1)) return summary def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) src = ap.add_mutually_exclusive_group(required=True) src.add_argument("--text", help="a .txt file: spoken and transcribed") src.add_argument("--video", help="an .mp4/.mov/.webm") src.add_argument("--audio", help="a .wav/.mp3") src.add_argument("--screen", help="a .png held still for --seconds") ap.add_argument("--seconds", type=float, default=20.0, help="how long a --screen is shown") ap.add_argument("--out", required=True) ap.add_argument("--cache", default=os.environ.get("TRIBE_CACHE", str(pathlib.Path.home() / ".cache/tribe-features"))) ap.add_argument("--no-map", action="store_true") args = ap.parse_args() kind = next(k for k in ("text", "video", "audio", "screen") if getattr(args, k)) cache = pathlib.Path(args.cache) cache.mkdir(parents=True, exist_ok=True) summary = encode_stimulus(load_model(cache), kind, getattr(args, kind), args.out, args.seconds, not args.no_map) print(json.dumps({k: summary[k] for k in ("segments", "seconds_predicted", "modalities", "runtime_s", "top_parcels")})) if __name__ == "__main__": main()