File size: 12,023 Bytes
6df569c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 | """Predict cortical responses to a stimulus with TRIBE v2 (facebook/tribev2, CC BY-NC 4.0).
A perception runs `encode_stimulus` on the model's inference endpoint (scripts/brain/tribe/handler.py,
docs/harnesses.md § Perception). The command line is the same computation, for studies run offline:
~/tribe-env/bin/python scripts/brain/encode.py --text page.txt --out run/
~/tribe-env/bin/python scripts/brain/encode.py --video clip.mp4 --out run/
~/tribe-env/bin/python scripts/brain/encode.py --screen frame.png --seconds 20 --out run/
A text is spoken (text-to-speech) and transcribed back to timed words, as TRIBE's own
demo does; a screen is a still held for `--seconds` as a silent video; a video or audio
file is taken as it is. The model answers one prediction a second (its TR) on the
fsaverage5 cortical mesh (20,484 vertices, left then right), for the average subject, in
z-units of BOLD, shifted 5 s into the past for the haemodynamic lag.
Writes into --out:
preds.npz preds (T, 20484) float16 and times (T,) seconds
rois.json every Destrieux parcel's time-mean and peak, per hemisphere, with its name
features.npz the backbone features the model was given (time-mean per modality), for
matched-control analyses: the same stimulus without the brain projection
map.png the time-mean prediction on the inflated surface, four views
summary.json what was run (model, checkpoint digest, stimulus, seconds, segments,
duration), the hemisphere means, and the ten strongest parcels
It fails plainly without the model, the GPU or the stimulus; it never writes a result it
did not compute.
"""
import argparse
import hashlib
import json
import os
import pathlib
import subprocess
import time
MODEL_ID = "facebook/tribev2"
MESH = "fsaverage5"
N_VERTICES = 20484
def sha256(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def ffmpeg():
import imageio_ffmpeg
return imageio_ffmpeg.get_ffmpeg_exe()
def screen_video(png, seconds, out):
"""A still image held for `seconds`, with a silent audio track so the audio extractor has one."""
subprocess.run([ffmpeg(), "-y", "-loglevel", "error", "-loop", "1", "-framerate", "4", "-i", str(png),
"-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", str(seconds),
"-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p", "-c:v", "libx264", "-preset", "fast", "-crf", "20",
"-c:a", "aac", "-shortest", str(out)], check=True)
return out
def transcribe_on_cpu():
"""TRIBE transcribes with whisperx under `uvx` on CUDA in float16; CTranslate2's aarch64 wheel has
no CUDA, so here whisperx runs on the CPU in int8 (BRAIN_WHISPER_DEVICE=cuda keeps TRIBE's own)."""
if os.environ.get("BRAIN_WHISPER_DEVICE", "cpu") == "cuda":
return
import json as _json
import subprocess as _sp
import tempfile
from pathlib import Path
import pandas as pd
from tribev2 import eventstransforms as et
codes = dict(english="en", french="fr", spanish="es", dutch="nl", chinese="zh")
def get_transcript(wav_filename, language):
if language not in codes:
raise ValueError(f"Language {language} not supported")
with tempfile.TemporaryDirectory() as out:
cmd = ["uvx", "whisperx", str(wav_filename), "--model", "large-v3", "--language", codes[language], "--device", "cpu",
"--compute_type", "int8", "--batch_size", "16", "--output_dir", out, "--output_format", "json"]
if language == "english":
cmd += ["--align_model", "WAV2VEC2_ASR_LARGE_LV60K_960H"]
env = {k: v for k, v in os.environ.items() if k != "MPLBACKEND"}
r = _sp.run(cmd, capture_output=True, text=True, env=env)
if r.returncode != 0:
raise RuntimeError(f"whisperx failed:\n{r.stderr[-2000:]}")
transcript = _json.loads((Path(out) / f"{Path(wav_filename).stem}.json").read_text())
words = []
for i, seg in enumerate(transcript["segments"]):
sentence = seg["text"].replace('"', "")
for w in seg["words"]:
if "start" in w:
words.append({"text": w["word"].replace('"', ""), "start": w["start"], "duration": w["end"] - w["start"], "sequence_id": i, "sentence": sentence})
return pd.DataFrame(words)
et.ExtractWordsFromAudio._get_transcript_from_audio = staticmethod(get_transcript)
def load_model(cache):
from tribev2.demo_utils import TribeModel
transcribe_on_cpu()
return TribeModel.from_pretrained(MODEL_ID, cache_folder=str(cache))
def checkpoint_digest():
from huggingface_hub import hf_hub_download
return sha256(hf_hub_download(MODEL_ID, "best.ckpt"))
class FeatureTap:
"""Catches the per-modality backbone features each batch carries into the model."""
def __init__(self, model):
self.sums, self.counts = {}, {}
self.handle = model.register_forward_pre_hook(self.hook, with_kwargs=False)
def hook(self, module, args):
import torch
batch = args[0]
for modality in ("text", "audio", "video"):
if modality not in batch.data:
continue
x = batch.data[modality].to(torch.float32) # B, L, D, T (or B, D, T)
if x.ndim == 3:
x = x.unsqueeze(1)
x = x.mean(dim=1) # mean over layers: B, D, T
self.sums[modality] = self.sums.get(modality, 0) + x.sum(dim=(0, 2)).cpu().numpy()
self.counts[modality] = self.counts.get(modality, 0) + x.shape[0] * x.shape[2]
def means(self):
return {m: self.sums[m] / max(self.counts[m], 1) for m in self.sums}
def atlas():
"""Destrieux parcels on fsaverage5: (labels_left, labels_right, names)."""
from nilearn import datasets
a = datasets.fetch_atlas_surf_destrieux()
names = [n.decode() if isinstance(n, bytes) else str(n) for n in a["labels"]]
return a["map_left"], a["map_right"], names
def parcels(mean, peak, labels_l, labels_r, names):
import numpy as np
out = []
half = N_VERTICES // 2
for hemi, labels, lo in (("L", labels_l, 0), ("R", labels_r, half)):
for idx, name in enumerate(names):
if name in ("Unknown", "Medial_wall"):
continue
sel = np.where(labels == idx)[0]
if not len(sel):
continue
out.append({"id": f"{hemi}_{name}", "hemisphere": hemi, "name": name, "vertices": int(len(sel)),
"mean": float(mean[lo + sel].mean()), "peak": float(peak[lo + sel].mean())})
return out
def plot_map(mean, out):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
from nilearn import datasets, plotting
fs = datasets.fetch_surf_fsaverage(MESH)
half = N_VERTICES // 2
v = float(np.percentile(np.abs(mean), 99)) or 1.0
fig, axes = plt.subplots(1, 4, figsize=(16, 4), subplot_kw={"projection": "3d"})
views = [("left", "lateral", mean[:half]), ("left", "medial", mean[:half]), ("right", "lateral", mean[half:]), ("right", "medial", mean[half:])]
for ax, (hemi, view, data) in zip(axes, views):
plotting.plot_surf_stat_map(fs[f"infl_{hemi}"], data, hemi=hemi, view=view, bg_map=fs[f"sulc_{hemi}"], cmap="RdBu_r",
vmax=v, symmetric_cbar=True, colorbar=(view == "medial" and hemi == "right"), axes=ax, threshold=v * 0.1)
ax.set_title(f"{hemi} {view}", fontsize=10)
fig.savefig(out, dpi=110, bbox_inches="tight")
plt.close(fig)
def encode_stimulus(model, kind, path, out, seconds=20.0, draw_map=True):
"""Predict the response to one stimulus with a loaded model, and write the files above into `out`.
`kind` is text, video, audio or screen; `path` the file. → the summary."""
import numpy as np
path = pathlib.Path(path)
out = pathlib.Path(out)
out.mkdir(parents=True, exist_ok=True)
t0 = time.time()
stimulus = {"kind": kind, "source": str(path), "sha256": sha256(path)}
if kind == "screen":
clip = out / "stimulus.mp4"
screen_video(path, seconds, clip)
stimulus["seconds"] = seconds
events_arg = {"video_path": str(clip)}
elif kind == "video":
events_arg = {"video_path": str(path.resolve())}
elif kind == "audio":
events_arg = {"audio_path": str(path.resolve())}
elif kind == "text":
stimulus["words"] = len(path.read_text(encoding="utf-8").split())
events_arg = {"text_path": str(path.resolve())}
else:
raise ValueError(f"no stimulus of the kind {kind}")
tap = FeatureTap(model._model)
events = model.get_events_dataframe(**events_arg)
preds, segments = model.predict(events, verbose=False)
tap.handle.remove()
if preds.shape[1] != N_VERTICES:
raise RuntimeError(f"expected {N_VERTICES} vertices, got {preds.shape[1]}")
times = np.array([float(s.start) for s in segments])
order = np.argsort(times)
preds, times = preds[order], times[order]
mean = preds.mean(axis=0)
peak = preds.max(axis=0)
np.savez_compressed(out / "preds.npz", preds=preds.astype(np.float16), times=times.astype(np.float32))
features = tap.means()
np.savez_compressed(out / "features.npz", **{k: v.astype(np.float32) for k, v in features.items()})
labels_l, labels_r, names = atlas()
rows = parcels(mean, peak, labels_l, labels_r, names)
(out / "rois.json").write_text(json.dumps({"atlas": "Destrieux 2009 on fsaverage5 (nilearn)", "parcels": rows}, indent=1))
if draw_map:
plot_map(mean, out / "map.png")
half = N_VERTICES // 2
top = sorted(rows, key=lambda r: -r["mean"])[:10]
summary = {
"model": MODEL_ID, "checkpoint_sha256": checkpoint_digest(), "mesh": MESH, "vertices": N_VERTICES,
"stimulus": stimulus, "segments": int(preds.shape[0]), "seconds_predicted": float(times[-1] - times[0] + 1) if len(times) else 0.0,
"modalities": sorted(features), "feature_dims": {k: int(v.shape[0]) for k, v in features.items()},
"hemisphere_mean": {"left": float(mean[:half].mean()), "right": float(mean[half:].mean())},
"global_mean": float(mean.mean()), "global_peak": float(peak.max()),
"top_parcels": [{"id": r["id"], "mean": r["mean"]} for r in top],
"runtime_s": round(time.time() - t0, 1),
"files": ["preds.npz", "rois.json", "features.npz"] + (["map.png"] if draw_map else []),
}
(out / "summary.json").write_text(json.dumps(summary, indent=1))
return summary
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
src = ap.add_mutually_exclusive_group(required=True)
src.add_argument("--text", help="a .txt file: spoken and transcribed")
src.add_argument("--video", help="an .mp4/.mov/.webm")
src.add_argument("--audio", help="a .wav/.mp3")
src.add_argument("--screen", help="a .png held still for --seconds")
ap.add_argument("--seconds", type=float, default=20.0, help="how long a --screen is shown")
ap.add_argument("--out", required=True)
ap.add_argument("--cache", default=os.environ.get("TRIBE_CACHE", str(pathlib.Path.home() / ".cache/tribe-features")))
ap.add_argument("--no-map", action="store_true")
args = ap.parse_args()
kind = next(k for k in ("text", "video", "audio", "screen") if getattr(args, k))
cache = pathlib.Path(args.cache)
cache.mkdir(parents=True, exist_ok=True)
summary = encode_stimulus(load_model(cache), kind, getattr(args, kind), args.out, args.seconds, not args.no_map)
print(json.dumps({k: summary[k] for k in ("segments", "seconds_predicted", "modalities", "runtime_s", "top_parcels")}))
if __name__ == "__main__":
main()
|