File size: 18,041 Bytes
75d5549 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 | infra:
folder: /checkpoint/sdascoli/results/tribe_release/half_depth/brain_model_config.subject_layers.subject_dropout=0.10,data.study.names=(Algonauts2025Bold,Lahner2024Bold,Lebel2023Bold,Wen2017)-ecdd89b2
cluster: auto
logs: '{folder}/logs/{user}/%j'
job_name: brain_model_config.subject_layers.subject_dropout=0.10,data.study.names=(Algonauts2025Bold,Lahner2024Bold,Lebel2023Bold,Wen2017)-ecdd89b2
timeout_min: 4320
nodes: 1
tasks_per_node: 1
cpus_per_task: 20
gpus_per_node: 1
mem_gb: 128.0
max_pickle_size_gb: null
slurm_constraint: volta32gb
slurm_partition: learnfair
slurm_account: null
slurm_qos: null
slurm_use_srun: false
slurm_additional_parameters: null
conda_env: null
workdir:
copied:
- neuralset
- neuraltrain
- neuralhub
- tribe
folder: null
log_commit: false
includes:
- '*.py'
- '*.txt'
excludes: !!python/tuple
- __pycache__
- .git
permissions: 511
version: '1'
mode: retry
keep_in_ram: false
data:
study:
infra: null
names:
- Algonauts2025Bold
- Lahner2024Bold
- Lebel2023Bold
- Wen2017
path: /large_experiments/brainai/shared/studies
transforms:
extractaudio:
infra: null
overwrite: false
name: ExtractAudioFromVideo
extractwords:
infra: null
language: english
overwrite: false
name: ExtractWordsFromAudio
addtext:
infra: null
name: AddText
addsentence:
infra: null
max_unmatched_ratio: 0.05
override_sentences: false
name: AddSentenceToWords
addcontext:
infra: null
sentence_only: false
max_context_len: 1024
split_field: ''
name: AddContextToWords
removemissing:
infra: null
event_types: Word
field: context
name: RemoveMissing
chunksounds:
infra: null
event_type_to_chunk: Audio
event_type_to_use: null
min_duration: 30.0
max_duration: 60.0
name: ChunkEvents
chunkvideos:
infra:
folder: !!python/object/apply:pathlib.PosixPath
- /
- checkpoint
- sdascoli
- cache
- tribe_release
cache_type: ValidatedParquet
mode: cached
keep_in_ram: false
backend: Cached
event_type_to_chunk: Video
event_type_to_use: null
min_duration: 30.0
max_duration: 60.0
name: ChunkEvents
query:
infra: null
query: null
name: QueryEvents
split:
infra: null
val_ratio: 0.1
name: SplitEvents
query: null
studies_to_include: null
infra_timelines:
folder: /checkpoint/sdascoli/cache/tribe_release
cluster: processpool
logs: '{folder}/logs/{user}/%j'
job_name: null
timeout_min: 720
nodes: 1
tasks_per_node: 1
cpus_per_task: null
gpus_per_node: null
mem_gb: null
max_pickle_size_gb: null
slurm_constraint: null
slurm_partition: null
slurm_account: null
slurm_qos: null
slurm_use_srun: false
slurm_additional_parameters: null
conda_env: null
workdir: null
permissions: 511
version: final
keep_in_ram: true
max_jobs: 1024
min_samples_per_job: 4
forbid_single_item_computation: false
mode: cached
name: MultiStudyLoader
neuro:
event_types: Fmri
aggregation: single
allow_missing: true
frequency: 1.0
offset: 5.0
projection:
mesh: fsaverage5
radius: 3.0
interpolation: linear
kind: ball
n_samples: null
mask_img: null
depth: null
center_depth: 0.5
extract_fsaverage_from_mni: true
name: TribeSurfaceProjector
cleaning:
standardize: zscore_sample
detrend: true
high_pass: null
low_pass: null
filter: null
ensure_finite: true
padding: null
input_space: null
fwhm: null
infra:
folder: /checkpoint/sdascoli/cache/tribe_release
cluster: slurm
logs: '{folder}/logs/{user}/%j'
job_name: null
timeout_min: 720
nodes: 1
tasks_per_node: 1
cpus_per_task: 10
gpus_per_node: 1
mem_gb: null
max_pickle_size_gb: null
slurm_constraint: volta32gb
slurm_partition: scavenge
slurm_account: null
slurm_qos: null
slurm_use_srun: false
slurm_additional_parameters: null
conda_env: null
workdir: null
permissions: 511
version: release
keep_in_ram: true
max_jobs: 256
min_samples_per_job: 1
forbid_single_item_computation: false
mode: cached
name: FmriExtractor
text_feature:
model_name: meta-llama/Llama-3.2-3B
device: cuda
layers:
- 0.5
- 0.75
- 1.0
cache_all_layers: false
cache_n_layers: 20
layer_aggregation: group_mean
token_aggregation: mean
event_types: Word
aggregation: sum
allow_missing: true
frequency: 2.0
infra:
folder: /checkpoint/sdascoli/cache/tribe_release
cluster: slurm
logs: '{folder}/logs/{user}/%j'
job_name: null
timeout_min: 720
nodes: 1
tasks_per_node: 1
cpus_per_task: 10
gpus_per_node: 1
mem_gb: null
max_pickle_size_gb: null
slurm_constraint: volta32gb
slurm_partition: scavenge
slurm_account: null
slurm_qos: null
slurm_use_srun: false
slurm_additional_parameters: null
conda_env: null
workdir: null
permissions: 511
version: release
keep_in_ram: true
max_jobs: 256
min_samples_per_job: 32
forbid_single_item_computation: false
mode: cached
batch_size: 4
contextualized: true
pretrained: true
name: HuggingFaceText
image_feature:
event_types: Video
aggregation: sum
allow_missing: true
frequency: 2.0
image:
model_name: facebook/dinov2-large
device: cuda
layers: 0.6666666666666666
cache_all_layers: false
cache_n_layers: 20
layer_aggregation: mean
token_aggregation: mean
event_types: Image
aggregation: single
allow_missing: false
frequency: 0.0
batch_size: 2
imsize: null
pretrained: true
infra:
folder: null
cluster: null
logs: '{folder}/logs/{user}/%j'
job_name: null
timeout_min: 25
nodes: 1
tasks_per_node: 1
cpus_per_task: 8
gpus_per_node: 1
mem_gb: null
max_pickle_size_gb: null
slurm_constraint: null
slurm_partition: null
slurm_account: null
slurm_qos: null
slurm_use_srun: false
slurm_additional_parameters: null
conda_env: null
workdir: null
permissions: 511
version: v5
keep_in_ram: false
max_jobs: 128
min_samples_per_job: 4096
forbid_single_item_computation: false
mode: cached
name: HuggingFaceImage
use_audio: true
clip_duration: null
max_imsize: null
layer_type: ''
num_frames: null
infra:
folder: /checkpoint/sdascoli/cache/tribe_release
cluster: slurm
logs: '{folder}/logs/{user}/%j'
job_name: null
timeout_min: 1440
nodes: 1
tasks_per_node: 1
cpus_per_task: 10
gpus_per_node: 1
mem_gb: null
max_pickle_size_gb: null
slurm_constraint: volta32gb
slurm_partition: scavenge
slurm_account: null
slurm_qos: null
slurm_use_srun: false
slurm_additional_parameters: null
conda_env: null
workdir: null
permissions: 511
version: release
keep_in_ram: true
max_jobs: 1024
min_samples_per_job: 1
forbid_single_item_computation: false
mode: cached
name: HuggingFaceVideo
audio_feature:
model_name: facebook/w2v-bert-2.0
device: cuda
layers:
- 0.5
- 0.75
- 1.0
cache_all_layers: false
cache_n_layers: 20
layer_aggregation: group_mean
token_aggregation: mean
event_types: Audio
aggregation: sum
allow_missing: true
frequency: 2.0
norm_audio: true
infra:
folder: /checkpoint/sdascoli/cache/tribe_release
cluster: slurm
logs: '{folder}/logs/{user}/%j'
job_name: null
timeout_min: 720
nodes: 1
tasks_per_node: 1
cpus_per_task: 10
gpus_per_node: 1
mem_gb: null
max_pickle_size_gb: null
slurm_constraint: volta32gb
slurm_partition: scavenge
slurm_account: null
slurm_qos: null
slurm_use_srun: false
slurm_additional_parameters: null
conda_env: null
workdir: null
permissions: 511
version: release
keep_in_ram: true
max_jobs: 256
min_samples_per_job: 1
forbid_single_item_computation: false
mode: cached
normalized: true
layer_type: transformer
name: Wav2VecBert
video_feature:
event_types: Video
aggregation: sum
allow_missing: true
frequency: 2.0
image:
model_name: facebook/vjepa2-vitg-fpc64-256
device: cuda
layers:
- 0.5
- 0.75
- 1.0
cache_all_layers: false
cache_n_layers: 20
layer_aggregation: group_mean
token_aggregation: mean
event_types: Image
aggregation: single
allow_missing: false
frequency: 0.0
batch_size: 8
imsize: null
pretrained: true
infra:
folder: null
cluster: null
logs: '{folder}/logs/{user}/%j'
job_name: null
timeout_min: 25
nodes: 1
tasks_per_node: 1
cpus_per_task: 8
gpus_per_node: 1
mem_gb: null
max_pickle_size_gb: null
slurm_constraint: null
slurm_partition: null
slurm_account: null
slurm_qos: null
slurm_use_srun: false
slurm_additional_parameters: null
conda_env: null
workdir: null
permissions: 511
version: v5
keep_in_ram: false
max_jobs: 128
min_samples_per_job: 4096
forbid_single_item_computation: false
mode: cached
name: HuggingFaceImage
use_audio: true
clip_duration: 4.0
max_imsize: null
layer_type: ''
num_frames: null
infra:
folder: /checkpoint/sdascoli/cache/tribe_release
cluster: slurm
logs: '{folder}/logs/{user}/%j'
job_name: null
timeout_min: 1440
nodes: 1
tasks_per_node: 1
cpus_per_task: 10
gpus_per_node: 1
mem_gb: null
max_pickle_size_gb: null
slurm_constraint: volta32gb
slurm_partition: scavenge
slurm_account: null
slurm_qos: null
slurm_use_srun: false
slurm_additional_parameters: null
conda_env: null
workdir: null
permissions: 511
version: release
keep_in_ram: true
max_jobs: 1024
min_samples_per_job: 1
forbid_single_item_computation: false
mode: cached
name: HuggingFaceVideo
subject_id:
event_types: Event
aggregation: first
allow_missing: true
frequency: 0.0
event_field: subject
treat_missing_as_separate_class: false
return_one_hot: false
predefined_mapping:
Algonauts2025Bold/sub-01: 0
Algonauts2025Bold/sub-02: 1
Algonauts2025Bold/sub-03: 2
Algonauts2025Bold/sub-05: 3
Lahner2024Bold/1: 4
Lahner2024Bold/10: 5
Lahner2024Bold/2: 6
Lahner2024Bold/3: 7
Lahner2024Bold/4: 8
Lahner2024Bold/5: 9
Lahner2024Bold/6: 10
Lahner2024Bold/7: 11
Lahner2024Bold/8: 12
Lahner2024Bold/9: 13
Lebel2023Bold/UTS01: 14
Lebel2023Bold/UTS02: 15
Lebel2023Bold/UTS03: 16
Lebel2023Bold/UTS04: 17
Lebel2023Bold/UTS05: 18
Lebel2023Bold/UTS06: 19
Lebel2023Bold/UTS07: 20
Lebel2023Bold/UTS08: 21
Wen2017/subject1: 22
Wen2017/subject2: 23
Wen2017/subject3: 24
name: LabelEncoder
frequency: 2.0
features_to_use:
- text
- audio
- video
features_to_mask: []
n_layers_to_use: null
layers_to_use:
- 0.5
- 0.75
- 1.0
layer_aggregation: group_mean
duration_trs: 100
overlap_trs_train: 0
overlap_trs_val: 0
batch_size: 8
num_workers: 20
shuffle_train: true
shuffle_val: true
stride_drop_incomplete: false
split_segments_by_time: false
seed: 33
brain_model_config:
projector:
input_size: null
hidden_sizes: null
norm_layer: layer
activation_layer: gelu
bias: true
dropout: 0.0
name: Mlp
combiner: null
encoder:
heads: 8
depth: 8
cross_attend: false
causal: false
attn_flash: false
attn_dropout: 0.0
ff_mult: 4
ff_dropout: 0.0
use_scalenorm: true
use_rmsnorm: false
rel_pos_bias: false
alibi_pos_bias: false
rotary_pos_emb: true
rotary_xpos: false
residual_attn: false
scale_residual: true
layer_dropout: 0.0
name: TransformerEncoder
time_pos_embedding: true
subject_embedding: false
subject_layers:
n_subjects: 25
bias: true
init_id: false
average_subjects: false
mode: gather
subject_dropout: 0.1
name: SubjectLayers
hidden: 1152
max_seq_len: 1024
dropout: 0.0
extractor_aggregation: cat
layer_aggregation: cat
linear_baseline: false
modality_dropout: 0.3
temporal_dropout: 0.0
low_rank_head: 2048
temporal_smoothing: null
name: FmriEncoder
loss:
kwargs:
reduction: none
name: MSELoss
optim:
optimizer:
lr: 0.0001
kwargs:
weight_decay: 0.0
name: Adam
scheduler:
kwargs:
max_lr: 0.0001
pct_start: 0.1
name: OneCycleLR
interval: step
name: LightningOptimizer
metrics:
- log_name: pearson
dim: 0
reduction: mean
torchmetrics_kwargs: null
name: OnlinePearsonCorr
- log_name: subj_pearson
metric_name: OnlinePearsonCorr
kwargs:
dim: 0
name: GroupedMetric
- log_name: retrieval_top1
topk: 1
name: TopkAcc
monitor: val/pearson
wandb_config:
name: brain_model_config.subject_layers.subject_dropout=0.10,data.study.names=(Algonauts2025Bold,Lahner2024Bold,Lebel2023Bold,Wen2017)-ecdd89b2
group: half_depth
entity: sdascoli
project: tribe_release
offline: false
host: https://fairwandb.org
id: null
dir: null
anonymous: null
log_model: false
experiment: null
prefix: ''
resume: allow
accelerator: gpu
n_epochs: 15
max_steps: -1
patience: null
limit_train_batches: null
accumulate_grad_batches: 1
enable_progress_bar: true
log_every_n_steps: 5
fast_dev_run: false
save_checkpoints: true
checkpoint_filename: best
resize_subject_layer: false
freeze_backbone: false
average_subjects: false
checkpoint_path: null
load_checkpoint: true
test_only: false
|