File size: 12,023 Bytes
6df569c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
"""Predict cortical responses to a stimulus with TRIBE v2 (facebook/tribev2, CC BY-NC 4.0).

A perception runs `encode_stimulus` on the model's inference endpoint (scripts/brain/tribe/handler.py,
docs/harnesses.md § Perception). The command line is the same computation, for studies run offline:

    ~/tribe-env/bin/python scripts/brain/encode.py --text page.txt --out run/
    ~/tribe-env/bin/python scripts/brain/encode.py --video clip.mp4 --out run/
    ~/tribe-env/bin/python scripts/brain/encode.py --screen frame.png --seconds 20 --out run/

A text is spoken (text-to-speech) and transcribed back to timed words, as TRIBE's own
demo does; a screen is a still held for `--seconds` as a silent video; a video or audio
file is taken as it is. The model answers one prediction a second (its TR) on the
fsaverage5 cortical mesh (20,484 vertices, left then right), for the average subject, in
z-units of BOLD, shifted 5 s into the past for the haemodynamic lag.

Writes into --out:

    preds.npz      preds (T, 20484) float16 and times (T,) seconds
    rois.json      every Destrieux parcel's time-mean and peak, per hemisphere, with its name
    features.npz   the backbone features the model was given (time-mean per modality), for
                   matched-control analyses: the same stimulus without the brain projection
    map.png        the time-mean prediction on the inflated surface, four views
    summary.json   what was run (model, checkpoint digest, stimulus, seconds, segments,
                   duration), the hemisphere means, and the ten strongest parcels

It fails plainly without the model, the GPU or the stimulus; it never writes a result it
did not compute.
"""

import argparse
import hashlib
import json
import os
import pathlib
import subprocess
import time

MODEL_ID = "facebook/tribev2"
MESH = "fsaverage5"
N_VERTICES = 20484


def sha256(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def ffmpeg():
    import imageio_ffmpeg
    return imageio_ffmpeg.get_ffmpeg_exe()


def screen_video(png, seconds, out):
    """A still image held for `seconds`, with a silent audio track so the audio extractor has one."""
    subprocess.run([ffmpeg(), "-y", "-loglevel", "error", "-loop", "1", "-framerate", "4", "-i", str(png),
                    "-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", str(seconds),
                    "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p", "-c:v", "libx264", "-preset", "fast", "-crf", "20",
                    "-c:a", "aac", "-shortest", str(out)], check=True)
    return out


def transcribe_on_cpu():
    """TRIBE transcribes with whisperx under `uvx` on CUDA in float16; CTranslate2's aarch64 wheel has
    no CUDA, so here whisperx runs on the CPU in int8 (BRAIN_WHISPER_DEVICE=cuda keeps TRIBE's own)."""
    if os.environ.get("BRAIN_WHISPER_DEVICE", "cpu") == "cuda":
        return
    import json as _json
    import subprocess as _sp
    import tempfile
    from pathlib import Path
    import pandas as pd
    from tribev2 import eventstransforms as et

    codes = dict(english="en", french="fr", spanish="es", dutch="nl", chinese="zh")

    def get_transcript(wav_filename, language):
        if language not in codes:
            raise ValueError(f"Language {language} not supported")
        with tempfile.TemporaryDirectory() as out:
            cmd = ["uvx", "whisperx", str(wav_filename), "--model", "large-v3", "--language", codes[language], "--device", "cpu",
                   "--compute_type", "int8", "--batch_size", "16", "--output_dir", out, "--output_format", "json"]
            if language == "english":
                cmd += ["--align_model", "WAV2VEC2_ASR_LARGE_LV60K_960H"]
            env = {k: v for k, v in os.environ.items() if k != "MPLBACKEND"}
            r = _sp.run(cmd, capture_output=True, text=True, env=env)
            if r.returncode != 0:
                raise RuntimeError(f"whisperx failed:\n{r.stderr[-2000:]}")
            transcript = _json.loads((Path(out) / f"{Path(wav_filename).stem}.json").read_text())
        words = []
        for i, seg in enumerate(transcript["segments"]):
            sentence = seg["text"].replace('"', "")
            for w in seg["words"]:
                if "start" in w:
                    words.append({"text": w["word"].replace('"', ""), "start": w["start"], "duration": w["end"] - w["start"], "sequence_id": i, "sentence": sentence})
        return pd.DataFrame(words)

    et.ExtractWordsFromAudio._get_transcript_from_audio = staticmethod(get_transcript)


def load_model(cache):
    from tribev2.demo_utils import TribeModel
    transcribe_on_cpu()
    return TribeModel.from_pretrained(MODEL_ID, cache_folder=str(cache))


def checkpoint_digest():
    from huggingface_hub import hf_hub_download
    return sha256(hf_hub_download(MODEL_ID, "best.ckpt"))


class FeatureTap:
    """Catches the per-modality backbone features each batch carries into the model."""

    def __init__(self, model):
        self.sums, self.counts = {}, {}
        self.handle = model.register_forward_pre_hook(self.hook, with_kwargs=False)

    def hook(self, module, args):
        import torch
        batch = args[0]
        for modality in ("text", "audio", "video"):
            if modality not in batch.data:
                continue
            x = batch.data[modality].to(torch.float32)  # B, L, D, T  (or B, D, T)
            if x.ndim == 3:
                x = x.unsqueeze(1)
            x = x.mean(dim=1)  # mean over layers: B, D, T
            self.sums[modality] = self.sums.get(modality, 0) + x.sum(dim=(0, 2)).cpu().numpy()
            self.counts[modality] = self.counts.get(modality, 0) + x.shape[0] * x.shape[2]

    def means(self):
        return {m: self.sums[m] / max(self.counts[m], 1) for m in self.sums}


def atlas():
    """Destrieux parcels on fsaverage5: (labels_left, labels_right, names)."""
    from nilearn import datasets
    a = datasets.fetch_atlas_surf_destrieux()
    names = [n.decode() if isinstance(n, bytes) else str(n) for n in a["labels"]]
    return a["map_left"], a["map_right"], names


def parcels(mean, peak, labels_l, labels_r, names):
    import numpy as np
    out = []
    half = N_VERTICES // 2
    for hemi, labels, lo in (("L", labels_l, 0), ("R", labels_r, half)):
        for idx, name in enumerate(names):
            if name in ("Unknown", "Medial_wall"):
                continue
            sel = np.where(labels == idx)[0]
            if not len(sel):
                continue
            out.append({"id": f"{hemi}_{name}", "hemisphere": hemi, "name": name, "vertices": int(len(sel)),
                        "mean": float(mean[lo + sel].mean()), "peak": float(peak[lo + sel].mean())})
    return out


def plot_map(mean, out):
    import matplotlib
    matplotlib.use("Agg")
    import matplotlib.pyplot as plt
    import numpy as np
    from nilearn import datasets, plotting
    fs = datasets.fetch_surf_fsaverage(MESH)
    half = N_VERTICES // 2
    v = float(np.percentile(np.abs(mean), 99)) or 1.0
    fig, axes = plt.subplots(1, 4, figsize=(16, 4), subplot_kw={"projection": "3d"})
    views = [("left", "lateral", mean[:half]), ("left", "medial", mean[:half]), ("right", "lateral", mean[half:]), ("right", "medial", mean[half:])]
    for ax, (hemi, view, data) in zip(axes, views):
        plotting.plot_surf_stat_map(fs[f"infl_{hemi}"], data, hemi=hemi, view=view, bg_map=fs[f"sulc_{hemi}"], cmap="RdBu_r",
                                    vmax=v, symmetric_cbar=True, colorbar=(view == "medial" and hemi == "right"), axes=ax, threshold=v * 0.1)
        ax.set_title(f"{hemi} {view}", fontsize=10)
    fig.savefig(out, dpi=110, bbox_inches="tight")
    plt.close(fig)


def encode_stimulus(model, kind, path, out, seconds=20.0, draw_map=True):
    """Predict the response to one stimulus with a loaded model, and write the files above into `out`.
    `kind` is text, video, audio or screen; `path` the file. → the summary."""
    import numpy as np
    path = pathlib.Path(path)
    out = pathlib.Path(out)
    out.mkdir(parents=True, exist_ok=True)
    t0 = time.time()
    stimulus = {"kind": kind, "source": str(path), "sha256": sha256(path)}
    if kind == "screen":
        clip = out / "stimulus.mp4"
        screen_video(path, seconds, clip)
        stimulus["seconds"] = seconds
        events_arg = {"video_path": str(clip)}
    elif kind == "video":
        events_arg = {"video_path": str(path.resolve())}
    elif kind == "audio":
        events_arg = {"audio_path": str(path.resolve())}
    elif kind == "text":
        stimulus["words"] = len(path.read_text(encoding="utf-8").split())
        events_arg = {"text_path": str(path.resolve())}
    else:
        raise ValueError(f"no stimulus of the kind {kind}")

    tap = FeatureTap(model._model)
    events = model.get_events_dataframe(**events_arg)
    preds, segments = model.predict(events, verbose=False)
    tap.handle.remove()
    if preds.shape[1] != N_VERTICES:
        raise RuntimeError(f"expected {N_VERTICES} vertices, got {preds.shape[1]}")
    times = np.array([float(s.start) for s in segments])
    order = np.argsort(times)
    preds, times = preds[order], times[order]
    mean = preds.mean(axis=0)
    peak = preds.max(axis=0)

    np.savez_compressed(out / "preds.npz", preds=preds.astype(np.float16), times=times.astype(np.float32))
    features = tap.means()
    np.savez_compressed(out / "features.npz", **{k: v.astype(np.float32) for k, v in features.items()})
    labels_l, labels_r, names = atlas()
    rows = parcels(mean, peak, labels_l, labels_r, names)
    (out / "rois.json").write_text(json.dumps({"atlas": "Destrieux 2009 on fsaverage5 (nilearn)", "parcels": rows}, indent=1))
    if draw_map:
        plot_map(mean, out / "map.png")
    half = N_VERTICES // 2
    top = sorted(rows, key=lambda r: -r["mean"])[:10]
    summary = {
        "model": MODEL_ID, "checkpoint_sha256": checkpoint_digest(), "mesh": MESH, "vertices": N_VERTICES,
        "stimulus": stimulus, "segments": int(preds.shape[0]), "seconds_predicted": float(times[-1] - times[0] + 1) if len(times) else 0.0,
        "modalities": sorted(features), "feature_dims": {k: int(v.shape[0]) for k, v in features.items()},
        "hemisphere_mean": {"left": float(mean[:half].mean()), "right": float(mean[half:].mean())},
        "global_mean": float(mean.mean()), "global_peak": float(peak.max()),
        "top_parcels": [{"id": r["id"], "mean": r["mean"]} for r in top],
        "runtime_s": round(time.time() - t0, 1),
        "files": ["preds.npz", "rois.json", "features.npz"] + (["map.png"] if draw_map else []),
    }
    (out / "summary.json").write_text(json.dumps(summary, indent=1))
    return summary


def main():
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    src = ap.add_mutually_exclusive_group(required=True)
    src.add_argument("--text", help="a .txt file: spoken and transcribed")
    src.add_argument("--video", help="an .mp4/.mov/.webm")
    src.add_argument("--audio", help="a .wav/.mp3")
    src.add_argument("--screen", help="a .png held still for --seconds")
    ap.add_argument("--seconds", type=float, default=20.0, help="how long a --screen is shown")
    ap.add_argument("--out", required=True)
    ap.add_argument("--cache", default=os.environ.get("TRIBE_CACHE", str(pathlib.Path.home() / ".cache/tribe-features")))
    ap.add_argument("--no-map", action="store_true")
    args = ap.parse_args()
    kind = next(k for k in ("text", "video", "audio", "screen") if getattr(args, k))
    cache = pathlib.Path(args.cache)
    cache.mkdir(parents=True, exist_ok=True)
    summary = encode_stimulus(load_model(cache), kind, getattr(args, kind), args.out, args.seconds, not args.no_map)
    print(json.dumps({k: summary[k] for k in ("segments", "seconds_predicted", "modalities", "runtime_s", "top_parcels")}))


if __name__ == "__main__":
    main()