Add files using upload-large-folder tool

ast_1_AS20k/ast_new_audioset/checkpoint_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22f4d1bdccbf34b3986cdfaf97bceaa9e82b4d3a8e011ea4e111904f294f8f6f
+size 352586874

ast_1_AS20k/ast_new_audioset/checkpoint_11.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:153f81afacd4d63f575871978aeb9d4d0bfdeb0b4a77c2ce2a1564dd07608579
+size 352587039

ast_1_AS20k/ast_new_audioset/checkpoint_15.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10cb65aeca1f773590b23ce9d2d705aead1e54a46f5fdf886d06f5aac1f3da41
+size 352587039

ast_1_AS20k/ast_new_audioset/checkpoint_16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebed3d5c168a32dadc61357b1a234ebf59651252ce1d5eb1880b6e469e82365b
+size 352587039

ast_1_AS20k/ast_new_audioset/checkpoint_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2be2bef5d1da4dcbbcf358260ca6f88b9322fdd3308399302c897cde1413f8f7
+size 352586874

ast_1_AS20k/ast_new_audioset/checkpoint_20.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c740289dd3129cdcb9e18ebbf228d47b9113f89cd5d4d0f5d6fe098e2a7ae9b6
+size 352587039

ast_1_AS20k/ast_new_audioset/checkpoint_21.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be020848a6bbca8a5eb8de7f3c10109fbc4284f5b9edb364d011ca4bddc6fa52
+size 352587039

ast_1_AS20k/ast_new_audioset/checkpoint_22.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:164f6312bb7956a80a31add7003edd9363dc9d69e28c76cdfd6223061bfc74c8
+size 352587039

ast_1_AS20k/ast_new_audioset/checkpoint_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34a58857d6206ff01e85982306ead2139a232cba9c49c334667a1893faf46abd
+size 352586874

ast_1_AS20k/ast_new_audioset/checkpoint_9.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7da4625fa3f5a8bd3adabf856aa377917ae7da7ee2f177d83ec8e66094d14ead
+size 352586874

ast_1_AS20k/ast_new_audioset/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d74a92edfb701a1e7fbfba743caab8d687ab122e679115e44d99b55809aa41ce
+size 352587369

ast_1_AS20k/ast_origin_implement/test-balanced-f10-t10-pTrue-b12-lr5e-5-decoupe/result.csv

@@ -0,0 +1,25 @@
+1.422225945558316940e-02,6.487563597807765037e-01,6.059869982241350601e-03,9.984290564287322534e-01,5.401801698178747557e-01,1.123867675634848823e-01,6.956494450569152832e-01,1.422225945558316940e-02,6.487563597807765037e-01,5.000000000000000240e-05
+5.325997280235295062e-02,8.158944676321755463e-01,1.306437393490925657e-02,9.709086275491799478e-01,1.272550891389265137e+00,2.355064642799161584e-02,6.952877044677734375e-01,5.093040520527827852e-02,8.065371915632957300e-01,5.000000000000000240e-05
+1.104590830848553862e-01,8.968618757921559270e-01,6.706614959781720398e-02,7.171915221978218957e-01,1.787383839223240622e+00,2.194607003870671647e-02,6.950482130050659180e-01,1.073419470179194091e-01,8.839559858178680507e-01,5.000000000000000240e-05
+1.848396644615896978e-01,9.313106471184702251e-01,7.167611495216880124e-02,6.694785396948753631e-01,2.100989061540869507e+00,2.017162883966943515e-02,6.948846578598022461e-01,1.791385835122070447e-01,9.215415158207399537e-01,5.000000000000000240e-05
+2.224462178579142690e-01,9.413168661608459775e-01,7.231485140869021999e-02,7.035087224749546619e-01,2.214551820998618137e+00,1.874961108785041033e-02,6.944540739059448242e-01,2.185575499842252467e-01,9.345964582357189077e-01,5.000000000000000240e-05
+2.548203316499514925e-01,9.479657052440092491e-01,5.870892769216756735e-02,7.725844616170880474e-01,2.298720947675281678e+00,1.763229149318959466e-02,6.945921778678894043e-01,2.526630268584063033e-01,9.435292535915005274e-01,5.000000000000000240e-05
+2.742160424132608632e-01,9.515682622442890315e-01,5.749621905254873044e-02,8.066989852343421363e-01,2.347952949601005201e+00,1.670030884553481282e-02,6.943714618682861328e-01,2.735843322137676004e-01,9.486307065585155573e-01,5.000000000000000240e-05
+2.891340671033661436e-01,9.539887857444313557e-01,5.727079787194905985e-02,8.220393549750605322e-01,2.382701716918723900e+00,1.596927918330596358e-02,6.942600607872009277e-01,2.902446482600923860e-01,9.520384123928512521e-01,5.000000000000000240e-05
+2.997937190939304331e-01,9.556058565055307596e-01,5.469515067152212751e-02,8.430501262804962481e-01,2.406743177850766635e+00,1.527043480806759661e-02,6.942504048347473145e-01,3.028899608327243476e-01,9.547648724123766195e-01,5.000000000000000240e-05
+3.019933285999398254e-01,9.551177780206882018e-01,5.673163717388598343e-02,8.369639127603817341e-01,2.399413530906429504e+00,1.471027910790956815e-02,6.941569447517395020e-01,3.115329036723915590e-01,9.565940278989387702e-01,5.000000000000000240e-05
+3.216990458954857579e-01,9.573572677069273063e-01,5.875842801928884279e-02,8.495200585745303901e-01,2.433591614166512151e+00,1.350063216294685418e-02,6.940920352935791016e-01,3.202949833204151719e-01,9.580394075942408882e-01,2.500000000000000120e-05
+3.237938459476596975e-01,9.568247991379073003e-01,5.627071898598238336e-02,8.472165610019362081e-01,2.425336344437045710e+00,1.277143365193674981e-02,6.940239071846008301e-01,3.264502469374453986e-01,9.591369993481135836e-01,2.500000000000000120e-05
+3.238792107002882448e-01,9.560063329207294514e-01,5.774346572337184930e-02,8.414383773149294310e-01,2.412805928578551029e+00,1.238875423823177294e-02,6.939673423767089844e-01,3.308619542913583955e-01,9.598927917725000869e-01,2.500000000000000120e-05
+3.206599597620702347e-01,9.540928202972318584e-01,5.840423268860757411e-02,8.374629739899399627e-01,2.384227802331127410e+00,1.193688203737030933e-02,6.939578056335449219e-01,3.341927086664225888e-01,9.604666643675140447e-01,2.500000000000000120e-05
+3.241879057377317075e-01,9.550509282205020822e-01,5.668243643471783388e-02,8.459431676912494424e-01,2.398414627994010839e+00,1.159679852697971178e-02,6.938989162445068359e-01,3.370616133181535967e-01,9.609689860314127863e-01,2.500000000000000120e-05
+3.274090426375390050e-01,9.538047907862127195e-01,5.882890954492734498e-02,8.395026159089084006e-01,2.380009463526119973e+00,1.087626258523142174e-02,6.938264966011047363e-01,3.390441351590992025e-01,9.612789191340032069e-01,1.250000000000000060e-05
+3.266298745594018449e-01,9.531561281365846794e-01,5.786502311129928383e-02,8.372581634183908772e-01,2.370586249390366440e+00,1.057763744308783116e-02,6.937884092330932617e-01,3.404189266645940570e-01,9.615116642607353104e-01,1.250000000000000060e-05
+3.272432009645968032e-01,9.523735520790914677e-01,5.759590126563685075e-02,8.398432031605835846e-01,2.359355951258224504e+00,1.041628441236315018e-02,6.938322186470031738e-01,3.419091536737948189e-01,9.616955903658975791e-01,1.250000000000000060e-05
+3.255807419210634546e-01,9.512071086194867631e-01,5.740117490775870773e-02,8.361371858623385389e-01,2.342888179552985672e+00,1.005493109991406793e-02,6.938048601150512695e-01,3.427275325615427026e-01,9.618161049645025384e-01,1.250000000000000060e-05
+3.260315090994287957e-01,9.512684697375038967e-01,5.997403778213243608e-02,8.288957385703106251e-01,2.343746581586883870e+00,9.951956874289869318e-03,6.936931610107421875e-01,3.433311674477271258e-01,9.618989983333274818e-01,1.250000000000000060e-05
+3.249040983002512983e-01,9.500949421687898688e-01,5.878458400191581557e-02,8.265363288594659297e-01,2.327477157438643030e+00,9.605016077793862572e-03,6.936856508255004883e-01,3.436332817873031242e-01,9.619345990419769787e-01,6.250000000000000300e-06
+3.255695187517096967e-01,9.499316465986847868e-01,5.797600592170489703e-02,8.279170449610905314e-01,2.325237544245208010e+00,9.480287065851862593e-03,6.936790347099304199e-01,3.441953690545042077e-01,9.619586781271860509e-01,6.250000000000000300e-06
+3.243047968373096723e-01,9.491688511518816540e-01,5.580597852971000417e-02,8.301318670729562754e-01,2.314852206236093224e+00,9.343003603446222577e-03,6.937055587768554688e-01,3.442617626297410083e-01,9.619489128942837475e-01,6.250000000000000300e-06
+3.250544555977981642e-01,9.489687673682015712e-01,5.894922958301676563e-02,8.262462971438052639e-01,2.312148661301067776e+00,9.364594131517802247e-03,6.936329603195190430e-01,3.443755436123956959e-01,9.619321524013675351e-01,6.250000000000000300e-06
+3.239834326417725396e-01,9.480251122897331850e-01,6.097054245045899906e-02,8.205525080094422385e-01,2.299510454271016968e+00,9.206688810194279052e-03,6.936240792274475098e-01,3.443998966946163476e-01,9.619017707066223055e-01,6.250000000000000300e-06

ast_1_AS20k/ast_origin_implement/test-balanced-f10-t10-pTrue-b12-lr5e-5-decoupe/train.log

@@ -0,0 +1,837 @@
+I am process 52222, running on zcs-cfc-eat-l-worker-0: starting (Wed Aug 27 02:40:44 2025)
+now train a audio spectrogram transformer model
+balanced sampler is not used
+---------------the train dataloader---------------
+now using following mask: 48 freq, 192 time
+now using mix-up with rate 0.500000
+now process audioset
+use dataset mean -4.268 and std 4.569 to normalize the input.
+number of classes is 527
+---------------the evaluation dataloader---------------
+now using following mask: 0 freq, 0 time
+now using mix-up with rate 0.000000
+now process audioset
+use dataset mean -4.268 and std 4.569 to normalize the input.
+number of classes is 527
+---------------AST Model Summary---------------
+ImageNet pretraining: True, AudioSet pretraining: False
+frequncey stride=10, time stride=10
+number of patches=1212
+
+Creating experiment directory: /opt/gpfs/home/chushu/exp/eat/ast_1_AS20k/ast_origin_implement/test-balanced-f10-t10-pTrue-b12-lr5e-5-decoupe
+Now starting training for 25 epochs
+running on cuda
+Total parameter number is : 88.132 million
+Total trainable parameter number is : 88.132 million
+now training with audioset, main metrics: mAP, loss function: BCEWithLogitsLoss(), learning rate scheduler: <torch.optim.lr_scheduler.MultiStepLR object at 0x7f99dcde0df0>
+The learning rate scheduler starts at 10 epoch with decay rate of 0.500 every 5 epochs
+current #steps=0, #epochs=1
+start training...
+---------------
+2025-08-27 02:40:45.711145
+current #epochs=1, #steps=0
+warm-up learning rate is 0.000000
+warm-up learning rate is 0.000003
+warm-up learning rate is 0.000005
+Epoch: [1][100/1713]	Per Sample Total Time 0.01385	Per Sample Data Time 0.00061	Per Sample DNN Time 0.01324	Train Loss 0.6809	
+warm-up learning rate is 0.000008
+warm-up learning rate is 0.000010
+Epoch: [1][200/1713]	Per Sample Total Time 0.01299	Per Sample Data Time 0.00033	Per Sample DNN Time 0.01267	Train Loss 0.5411	
+warm-up learning rate is 0.000013
+warm-up learning rate is 0.000015
+Epoch: [1][300/1713]	Per Sample Total Time 0.01253	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01230	Train Loss 0.4430	
+warm-up learning rate is 0.000017
+warm-up learning rate is 0.000020
+Epoch: [1][400/1713]	Per Sample Total Time 0.01241	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01222	Train Loss 0.3676	
+warm-up learning rate is 0.000023
+warm-up learning rate is 0.000025
+Epoch: [1][500/1713]	Per Sample Total Time 0.01238	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01223	Train Loss 0.3109	
+warm-up learning rate is 0.000028
+warm-up learning rate is 0.000030
+Epoch: [1][600/1713]	Per Sample Total Time 0.01228	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01215	Train Loss 0.2684	
+warm-up learning rate is 0.000033
+warm-up learning rate is 0.000035
+Epoch: [1][700/1713]	Per Sample Total Time 0.01227	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01215	Train Loss 0.2360	
+warm-up learning rate is 0.000038
+warm-up learning rate is 0.000040
+Epoch: [1][800/1713]	Per Sample Total Time 0.01229	Per Sample Data Time 0.00011	Per Sample DNN Time 0.01218	Train Loss 0.2108	
+warm-up learning rate is 0.000043
+warm-up learning rate is 0.000045
+Epoch: [1][900/1713]	Per Sample Total Time 0.01228	Per Sample Data Time 0.00011	Per Sample DNN Time 0.01217	Train Loss 0.1908	
+warm-up learning rate is 0.000048
+warm-up learning rate is 0.000050
+Epoch: [1][1000/1713]	Per Sample Total Time 0.01227	Per Sample Data Time 0.00010	Per Sample DNN Time 0.01217	Train Loss 0.1745	
+Epoch: [1][1100/1713]	Per Sample Total Time 0.01217	Per Sample Data Time 0.00009	Per Sample DNN Time 0.01208	Train Loss 0.1610	
+Epoch: [1][1200/1713]	Per Sample Total Time 0.01217	Per Sample Data Time 0.00009	Per Sample DNN Time 0.01208	Train Loss 0.1497	
+Epoch: [1][1300/1713]	Per Sample Total Time 0.01218	Per Sample Data Time 0.00009	Per Sample DNN Time 0.01209	Train Loss 0.1402	
+Epoch: [1][1400/1713]	Per Sample Total Time 0.01217	Per Sample Data Time 0.00008	Per Sample DNN Time 0.01209	Train Loss 0.1319	
+Epoch: [1][1500/1713]	Per Sample Total Time 0.01214	Per Sample Data Time 0.00008	Per Sample DNN Time 0.01206	Train Loss 0.1248	
+Epoch: [1][1600/1713]	Per Sample Total Time 0.01211	Per Sample Data Time 0.00008	Per Sample DNN Time 0.01203	Train Loss 0.1185	
+Epoch: [1][1700/1713]	Per Sample Total Time 0.01205	Per Sample Data Time 0.00007	Per Sample DNN Time 0.01198	Train Loss 0.1130	
+start validation
+mAP: 0.014222
+AUC: 0.648756
+Avg Precision: 0.006060
+Avg Recall: 0.998429
+d_prime: 0.540180
+train_loss: 0.112387
+valid_loss: 0.695649
+validation finished
+Epoch-1 lr: 5e-05
+epoch 1 training time: 327.615
+---------------
+2025-08-27 02:46:13.326272
+current #epochs=2, #steps=1713
+Epoch: [2][87/1713]	Per Sample Total Time 0.01450	Per Sample Data Time 0.00147	Per Sample DNN Time 0.01303	Train Loss 0.0242	
+Epoch: [2][187/1713]	Per Sample Total Time 0.01221	Per Sample Data Time 0.00071	Per Sample DNN Time 0.01150	Train Loss 0.0241	
+Epoch: [2][287/1713]	Per Sample Total Time 0.01223	Per Sample Data Time 0.00048	Per Sample DNN Time 0.01175	Train Loss 0.0241	
+Epoch: [2][387/1713]	Per Sample Total Time 0.01234	Per Sample Data Time 0.00036	Per Sample DNN Time 0.01198	Train Loss 0.0242	
+Epoch: [2][487/1713]	Per Sample Total Time 0.01244	Per Sample Data Time 0.00030	Per Sample DNN Time 0.01214	Train Loss 0.0241	
+Epoch: [2][587/1713]	Per Sample Total Time 0.01185	Per Sample Data Time 0.00025	Per Sample DNN Time 0.01159	Train Loss 0.0240	
+Epoch: [2][687/1713]	Per Sample Total Time 0.01197	Per Sample Data Time 0.00022	Per Sample DNN Time 0.01174	Train Loss 0.0240	
+Epoch: [2][787/1713]	Per Sample Total Time 0.01209	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01189	Train Loss 0.0239	
+Epoch: [2][887/1713]	Per Sample Total Time 0.01219	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01200	Train Loss 0.0239	
+Epoch: [2][987/1713]	Per Sample Total Time 0.01225	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01208	Train Loss 0.0239	
+Epoch: [2][1087/1713]	Per Sample Total Time 0.01196	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01181	Train Loss 0.0238	
+Epoch: [2][1187/1713]	Per Sample Total Time 0.01201	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01187	Train Loss 0.0238	
+Epoch: [2][1287/1713]	Per Sample Total Time 0.01205	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01190	Train Loss 0.0237	
+Epoch: [2][1387/1713]	Per Sample Total Time 0.01206	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01193	Train Loss 0.0237	
+Epoch: [2][1487/1713]	Per Sample Total Time 0.01183	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01171	Train Loss 0.0236	
+Epoch: [2][1587/1713]	Per Sample Total Time 0.01184	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01171	Train Loss 0.0236	
+Epoch: [2][1687/1713]	Per Sample Total Time 0.01187	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01175	Train Loss 0.0236	
+start validation
+mAP: 0.053260
+AUC: 0.815894
+Avg Precision: 0.013064
+Avg Recall: 0.970909
+d_prime: 1.272551
+train_loss: 0.023551
+valid_loss: 0.695288
+validation finished
+Epoch-2 lr: 5e-05
+epoch 2 training time: 324.709
+---------------
+2025-08-27 02:51:38.034980
+current #epochs=3, #steps=3426
+Epoch: [3][74/1713]	Per Sample Total Time 0.01292	Per Sample Data Time 0.00173	Per Sample DNN Time 0.01119	Train Loss 0.0230	
+Epoch: [3][174/1713]	Per Sample Total Time 0.01254	Per Sample Data Time 0.00077	Per Sample DNN Time 0.01177	Train Loss 0.0228	
+Epoch: [3][274/1713]	Per Sample Total Time 0.01243	Per Sample Data Time 0.00050	Per Sample DNN Time 0.01192	Train Loss 0.0228	
+Epoch: [3][374/1713]	Per Sample Total Time 0.01241	Per Sample Data Time 0.00038	Per Sample DNN Time 0.01203	Train Loss 0.0228	
+Epoch: [3][474/1713]	Per Sample Total Time 0.01191	Per Sample Data Time 0.00031	Per Sample DNN Time 0.01160	Train Loss 0.0226	
+Epoch: [3][574/1713]	Per Sample Total Time 0.01196	Per Sample Data Time 0.00026	Per Sample DNN Time 0.01169	Train Loss 0.0226	
+Epoch: [3][674/1713]	Per Sample Total Time 0.01200	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01177	Train Loss 0.0226	
+Epoch: [3][774/1713]	Per Sample Total Time 0.01170	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01149	Train Loss 0.0225	
+Epoch: [3][874/1713]	Per Sample Total Time 0.01170	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01151	Train Loss 0.0224	
+Epoch: [3][974/1713]	Per Sample Total Time 0.01178	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01160	Train Loss 0.0223	
+Epoch: [3][1074/1713]	Per Sample Total Time 0.01184	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01168	Train Loss 0.0223	
+Epoch: [3][1174/1713]	Per Sample Total Time 0.01168	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01153	Train Loss 0.0222	
+Epoch: [3][1274/1713]	Per Sample Total Time 0.01166	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01152	Train Loss 0.0221	
+Epoch: [3][1374/1713]	Per Sample Total Time 0.01173	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01160	Train Loss 0.0221	
+Epoch: [3][1474/1713]	Per Sample Total Time 0.01174	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01161	Train Loss 0.0221	
+Epoch: [3][1574/1713]	Per Sample Total Time 0.01178	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01165	Train Loss 0.0220	
+Epoch: [3][1674/1713]	Per Sample Total Time 0.01168	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01156	Train Loss 0.0220	
+start validation
+mAP: 0.110459
+AUC: 0.896862
+Avg Precision: 0.067066
+Avg Recall: 0.717192
+d_prime: 1.787384
+train_loss: 0.021946
+valid_loss: 0.695048
+validation finished
+Epoch-3 lr: 5e-05
+epoch 3 training time: 324.807
+---------------
+2025-08-27 02:57:02.842395
+current #epochs=4, #steps=5139
+Epoch: [4][61/1713]	Per Sample Total Time 0.01467	Per Sample Data Time 0.00201	Per Sample DNN Time 0.01265	Train Loss 0.0211	
+Epoch: [4][161/1713]	Per Sample Total Time 0.01335	Per Sample Data Time 0.00080	Per Sample DNN Time 0.01255	Train Loss 0.0209	
+Epoch: [4][261/1713]	Per Sample Total Time 0.01297	Per Sample Data Time 0.00051	Per Sample DNN Time 0.01245	Train Loss 0.0208	
+Epoch: [4][361/1713]	Per Sample Total Time 0.01263	Per Sample Data Time 0.00038	Per Sample DNN Time 0.01225	Train Loss 0.0209	
+Epoch: [4][461/1713]	Per Sample Total Time 0.01237	Per Sample Data Time 0.00031	Per Sample DNN Time 0.01206	Train Loss 0.0208	
+Epoch: [4][561/1713]	Per Sample Total Time 0.01234	Per Sample Data Time 0.00026	Per Sample DNN Time 0.01208	Train Loss 0.0207	
+Epoch: [4][661/1713]	Per Sample Total Time 0.01232	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01209	Train Loss 0.0207	
+Epoch: [4][761/1713]	Per Sample Total Time 0.01233	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01213	Train Loss 0.0207	
+Epoch: [4][861/1713]	Per Sample Total Time 0.01227	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01208	Train Loss 0.0206	
+Epoch: [4][961/1713]	Per Sample Total Time 0.01222	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01205	Train Loss 0.0205	
+Epoch: [4][1061/1713]	Per Sample Total Time 0.01197	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01181	Train Loss 0.0205	
+Epoch: [4][1161/1713]	Per Sample Total Time 0.01199	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01185	Train Loss 0.0204	
+Epoch: [4][1261/1713]	Per Sample Total Time 0.01204	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01190	Train Loss 0.0204	
+Epoch: [4][1361/1713]	Per Sample Total Time 0.01205	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01192	Train Loss 0.0203	
+Epoch: [4][1461/1713]	Per Sample Total Time 0.01204	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01192	Train Loss 0.0203	
+Epoch: [4][1561/1713]	Per Sample Total Time 0.01199	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01187	Train Loss 0.0202	
+Epoch: [4][1661/1713]	Per Sample Total Time 0.01202	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01190	Train Loss 0.0202	
+start validation
+mAP: 0.184840
+AUC: 0.931311
+Avg Precision: 0.071676
+Avg Recall: 0.669479
+d_prime: 2.100989
+train_loss: 0.020172
+valid_loss: 0.694885
+validation finished
+Epoch-4 lr: 5e-05
+epoch 4 training time: 330.929
+---------------
+2025-08-27 03:02:33.771092
+current #epochs=5, #steps=6852
+Epoch: [5][48/1713]	Per Sample Total Time 0.01044	Per Sample Data Time 0.00242	Per Sample DNN Time 0.00802	Train Loss 0.0193	
+Epoch: [5][148/1713]	Per Sample Total Time 0.01150	Per Sample Data Time 0.00082	Per Sample DNN Time 0.01068	Train Loss 0.0191	
+Epoch: [5][248/1713]	Per Sample Total Time 0.01172	Per Sample Data Time 0.00051	Per Sample DNN Time 0.01121	Train Loss 0.0190	
+Epoch: [5][348/1713]	Per Sample Total Time 0.01187	Per Sample Data Time 0.00038	Per Sample DNN Time 0.01149	Train Loss 0.0190	
+Epoch: [5][448/1713]	Per Sample Total Time 0.01133	Per Sample Data Time 0.00030	Per Sample DNN Time 0.01103	Train Loss 0.0190	
+Epoch: [5][548/1713]	Per Sample Total Time 0.01143	Per Sample Data Time 0.00025	Per Sample DNN Time 0.01118	Train Loss 0.0191	
+Epoch: [5][648/1713]	Per Sample Total Time 0.01154	Per Sample Data Time 0.00022	Per Sample DNN Time 0.01132	Train Loss 0.0190	
+Epoch: [5][748/1713]	Per Sample Total Time 0.01166	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01146	Train Loss 0.0189	
+Epoch: [5][848/1713]	Per Sample Total Time 0.01167	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01149	Train Loss 0.0189	
+Epoch: [5][948/1713]	Per Sample Total Time 0.01143	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01126	Train Loss 0.0190	
+Epoch: [5][1048/1713]	Per Sample Total Time 0.01153	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01138	Train Loss 0.0189	
+Epoch: [5][1148/1713]	Per Sample Total Time 0.01166	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01152	Train Loss 0.0189	
+Epoch: [5][1248/1713]	Per Sample Total Time 0.01169	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01155	Train Loss 0.0189	
+Epoch: [5][1348/1713]	Per Sample Total Time 0.01164	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01151	Train Loss 0.0189	
+Epoch: [5][1448/1713]	Per Sample Total Time 0.01169	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01157	Train Loss 0.0188	
+Epoch: [5][1548/1713]	Per Sample Total Time 0.01173	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01161	Train Loss 0.0188	
+Epoch: [5][1648/1713]	Per Sample Total Time 0.01176	Per Sample Data Time 0.00011	Per Sample DNN Time 0.01164	Train Loss 0.0187	
+start validation
+mAP: 0.222446
+AUC: 0.941317
+Avg Precision: 0.072315
+Avg Recall: 0.703509
+d_prime: 2.214552
+train_loss: 0.018750
+valid_loss: 0.694454
+validation finished
+Epoch-5 lr: 5e-05
+epoch 5 training time: 325.961
+---------------
+2025-08-27 03:07:59.731986
+current #epochs=6, #steps=8565
+Epoch: [6][35/1713]	Per Sample Total Time 0.01637	Per Sample Data Time 0.00389	Per Sample DNN Time 0.01247	Train Loss 0.0174	
+Epoch: [6][135/1713]	Per Sample Total Time 0.01327	Per Sample Data Time 0.00107	Per Sample DNN Time 0.01220	Train Loss 0.0180	
+Epoch: [6][235/1713]	Per Sample Total Time 0.01290	Per Sample Data Time 0.00064	Per Sample DNN Time 0.01226	Train Loss 0.0179	
+Epoch: [6][335/1713]	Per Sample Total Time 0.01226	Per Sample Data Time 0.00046	Per Sample DNN Time 0.01180	Train Loss 0.0179	
+Epoch: [6][435/1713]	Per Sample Total Time 0.01228	Per Sample Data Time 0.00036	Per Sample DNN Time 0.01191	Train Loss 0.0179	
+Epoch: [6][535/1713]	Per Sample Total Time 0.01234	Per Sample Data Time 0.00030	Per Sample DNN Time 0.01203	Train Loss 0.0178	
+Epoch: [6][635/1713]	Per Sample Total Time 0.01232	Per Sample Data Time 0.00026	Per Sample DNN Time 0.01205	Train Loss 0.0178	
+Epoch: [6][735/1713]	Per Sample Total Time 0.01214	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01191	Train Loss 0.0178	
+Epoch: [6][835/1713]	Per Sample Total Time 0.01211	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01190	Train Loss 0.0178	
+Epoch: [6][935/1713]	Per Sample Total Time 0.01213	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01193	Train Loss 0.0177	
+Epoch: [6][1035/1713]	Per Sample Total Time 0.01215	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01197	Train Loss 0.0177	
+Epoch: [6][1135/1713]	Per Sample Total Time 0.01212	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01196	Train Loss 0.0177	
+Epoch: [6][1235/1713]	Per Sample Total Time 0.01210	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01195	Train Loss 0.0177	
+Epoch: [6][1335/1713]	Per Sample Total Time 0.01211	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01196	Train Loss 0.0177	
+Epoch: [6][1435/1713]	Per Sample Total Time 0.01212	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01197	Train Loss 0.0177	
+Epoch: [6][1535/1713]	Per Sample Total Time 0.01212	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01198	Train Loss 0.0177	
+Epoch: [6][1635/1713]	Per Sample Total Time 0.01210	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01197	Train Loss 0.0176	
+start validation
+mAP: 0.254820
+AUC: 0.947966
+Avg Precision: 0.058709
+Avg Recall: 0.772584
+d_prime: 2.298721
+train_loss: 0.017632
+valid_loss: 0.694592
+validation finished
+Epoch-6 lr: 5e-05
+epoch 6 training time: 334.468
+---------------
+2025-08-27 03:13:34.200040
+current #epochs=7, #steps=10278
+Epoch: [7][22/1713]	Per Sample Total Time 0.01837	Per Sample Data Time 0.00587	Per Sample DNN Time 0.01250	Train Loss 0.0174	
+Epoch: [7][122/1713]	Per Sample Total Time 0.01342	Per Sample Data Time 0.00113	Per Sample DNN Time 0.01229	Train Loss 0.0168	
+Epoch: [7][222/1713]	Per Sample Total Time 0.01290	Per Sample Data Time 0.00065	Per Sample DNN Time 0.01225	Train Loss 0.0167	
+Epoch: [7][322/1713]	Per Sample Total Time 0.01259	Per Sample Data Time 0.00046	Per Sample DNN Time 0.01213	Train Loss 0.0167	
+Epoch: [7][422/1713]	Per Sample Total Time 0.01244	Per Sample Data Time 0.00036	Per Sample DNN Time 0.01208	Train Loss 0.0167	
+Epoch: [7][522/1713]	Per Sample Total Time 0.01226	Per Sample Data Time 0.00030	Per Sample DNN Time 0.01196	Train Loss 0.0168	
+Epoch: [7][622/1713]	Per Sample Total Time 0.01228	Per Sample Data Time 0.00026	Per Sample DNN Time 0.01202	Train Loss 0.0167	
+Epoch: [7][722/1713]	Per Sample Total Time 0.01201	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01178	Train Loss 0.0167	
+Epoch: [7][822/1713]	Per Sample Total Time 0.01205	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01184	Train Loss 0.0168	
+Epoch: [7][922/1713]	Per Sample Total Time 0.01210	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01191	Train Loss 0.0168	
+Epoch: [7][1022/1713]	Per Sample Total Time 0.01213	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01195	Train Loss 0.0168	
+Epoch: [7][1122/1713]	Per Sample Total Time 0.01215	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01199	Train Loss 0.0168	
+Epoch: [7][1222/1713]	Per Sample Total Time 0.01218	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01202	Train Loss 0.0167	
+Epoch: [7][1322/1713]	Per Sample Total Time 0.01219	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01205	Train Loss 0.0167	
+Epoch: [7][1422/1713]	Per Sample Total Time 0.01221	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01207	Train Loss 0.0167	
+Epoch: [7][1522/1713]	Per Sample Total Time 0.01222	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01209	Train Loss 0.0167	
+Epoch: [7][1622/1713]	Per Sample Total Time 0.01211	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01199	Train Loss 0.0167	
+start validation
+mAP: 0.274216
+AUC: 0.951568
+Avg Precision: 0.057496
+Avg Recall: 0.806699
+d_prime: 2.347953
+train_loss: 0.016700
+valid_loss: 0.694371
+validation finished
+Epoch-7 lr: 5e-05
+epoch 7 training time: 338.084
+---------------
+2025-08-27 03:19:12.284334
+current #epochs=8, #steps=11991
+Epoch: [8][9/1713]	Per Sample Total Time 0.02563	Per Sample Data Time 0.01307	Per Sample DNN Time 0.01255	Train Loss 0.0155	
+Epoch: [8][109/1713]	Per Sample Total Time 0.01330	Per Sample Data Time 0.00123	Per Sample DNN Time 0.01207	Train Loss 0.0156	
+Epoch: [8][209/1713]	Per Sample Total Time 0.01231	Per Sample Data Time 0.00066	Per Sample DNN Time 0.01165	Train Loss 0.0157	
+Epoch: [8][309/1713]	Per Sample Total Time 0.01212	Per Sample Data Time 0.00047	Per Sample DNN Time 0.01166	Train Loss 0.0158	
+Epoch: [8][409/1713]	Per Sample Total Time 0.01208	Per Sample Data Time 0.00036	Per Sample DNN Time 0.01172	Train Loss 0.0159	
+Epoch: [8][509/1713]	Per Sample Total Time 0.01211	Per Sample Data Time 0.00030	Per Sample DNN Time 0.01181	Train Loss 0.0159	
+Epoch: [8][609/1713]	Per Sample Total Time 0.01189	Per Sample Data Time 0.00026	Per Sample DNN Time 0.01163	Train Loss 0.0159	
+Epoch: [8][709/1713]	Per Sample Total Time 0.01173	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01150	Train Loss 0.0159	
+Epoch: [8][809/1713]	Per Sample Total Time 0.01175	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01154	Train Loss 0.0159	
+Epoch: [8][909/1713]	Per Sample Total Time 0.01177	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01158	Train Loss 0.0160	
+Epoch: [8][1009/1713]	Per Sample Total Time 0.01182	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01165	Train Loss 0.0159	
+Epoch: [8][1109/1713]	Per Sample Total Time 0.01170	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01154	Train Loss 0.0160	
+Epoch: [8][1209/1713]	Per Sample Total Time 0.01175	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01160	Train Loss 0.0160	
+Epoch: [8][1309/1713]	Per Sample Total Time 0.01179	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01164	Train Loss 0.0160	
+Epoch: [8][1409/1713]	Per Sample Total Time 0.01180	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01167	Train Loss 0.0160	
+Epoch: [8][1509/1713]	Per Sample Total Time 0.01171	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01159	Train Loss 0.0160	
+Epoch: [8][1609/1713]	Per Sample Total Time 0.01174	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01162	Train Loss 0.0160	
+Epoch: [8][1709/1713]	Per Sample Total Time 0.01177	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01165	Train Loss 0.0160	
+start validation
+mAP: 0.289134
+AUC: 0.953989
+Avg Precision: 0.057271
+Avg Recall: 0.822039
+d_prime: 2.382702
+train_loss: 0.015969
+valid_loss: 0.694260
+validation finished
+Epoch-8 lr: 5e-05
+epoch 8 training time: 326.545
+---------------
+2025-08-27 03:24:38.829389
+current #epochs=9, #steps=13704
+Epoch: [9][96/1713]	Per Sample Total Time 0.01018	Per Sample Data Time 0.00133	Per Sample DNN Time 0.00884	Train Loss 0.0148	
+Epoch: [9][196/1713]	Per Sample Total Time 0.01046	Per Sample Data Time 0.00067	Per Sample DNN Time 0.00978	Train Loss 0.0150	
+Epoch: [9][296/1713]	Per Sample Total Time 0.01110	Per Sample Data Time 0.00046	Per Sample DNN Time 0.01064	Train Loss 0.0150	
+Epoch: [9][396/1713]	Per Sample Total Time 0.01141	Per Sample Data Time 0.00036	Per Sample DNN Time 0.01106	Train Loss 0.0150	
+Epoch: [9][496/1713]	Per Sample Total Time 0.01171	Per Sample Data Time 0.00029	Per Sample DNN Time 0.01142	Train Loss 0.0150	
+Epoch: [9][596/1713]	Per Sample Total Time 0.01158	Per Sample Data Time 0.00025	Per Sample DNN Time 0.01132	Train Loss 0.0151	
+Epoch: [9][696/1713]	Per Sample Total Time 0.01148	Per Sample Data Time 0.00022	Per Sample DNN Time 0.01126	Train Loss 0.0151	
+Epoch: [9][796/1713]	Per Sample Total Time 0.01162	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01142	Train Loss 0.0152	
+Epoch: [9][896/1713]	Per Sample Total Time 0.01169	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01151	Train Loss 0.0152	
+Epoch: [9][996/1713]	Per Sample Total Time 0.01175	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01159	Train Loss 0.0152	
+Epoch: [9][1096/1713]	Per Sample Total Time 0.01171	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01155	Train Loss 0.0152	
+Epoch: [9][1196/1713]	Per Sample Total Time 0.01167	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01153	Train Loss 0.0152	
+Epoch: [9][1296/1713]	Per Sample Total Time 0.01176	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01162	Train Loss 0.0152	
+Epoch: [9][1396/1713]	Per Sample Total Time 0.01179	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01166	Train Loss 0.0152	
+Epoch: [9][1496/1713]	Per Sample Total Time 0.01179	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01166	Train Loss 0.0153	
+Epoch: [9][1596/1713]	Per Sample Total Time 0.01184	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01172	Train Loss 0.0153	
+Epoch: [9][1696/1713]	Per Sample Total Time 0.01188	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01176	Train Loss 0.0153	
+start validation
+mAP: 0.299794
+AUC: 0.955606
+Avg Precision: 0.054695
+Avg Recall: 0.843050
+d_prime: 2.406743
+train_loss: 0.015270
+valid_loss: 0.694250
+validation finished
+Epoch-9 lr: 5e-05
+epoch 9 training time: 328.892
+---------------
+2025-08-27 03:30:07.721315
+current #epochs=10, #steps=15417
+Epoch: [10][83/1713]	Per Sample Total Time 0.01054	Per Sample Data Time 0.00211	Per Sample DNN Time 0.00843	Train Loss 0.0144	
+Epoch: [10][183/1713]	Per Sample Total Time 0.01190	Per Sample Data Time 0.00099	Per Sample DNN Time 0.01090	Train Loss 0.0145	
+Epoch: [10][283/1713]	Per Sample Total Time 0.01228	Per Sample Data Time 0.00066	Per Sample DNN Time 0.01162	Train Loss 0.0145	
+Epoch: [10][383/1713]	Per Sample Total Time 0.01242	Per Sample Data Time 0.00050	Per Sample DNN Time 0.01192	Train Loss 0.0146	
+Epoch: [10][483/1713]	Per Sample Total Time 0.01206	Per Sample Data Time 0.00040	Per Sample DNN Time 0.01166	Train Loss 0.0146	
+Epoch: [10][583/1713]	Per Sample Total Time 0.01188	Per Sample Data Time 0.00034	Per Sample DNN Time 0.01154	Train Loss 0.0146	
+Epoch: [10][683/1713]	Per Sample Total Time 0.01195	Per Sample Data Time 0.00030	Per Sample DNN Time 0.01165	Train Loss 0.0146	
+Epoch: [10][783/1713]	Per Sample Total Time 0.01200	Per Sample Data Time 0.00027	Per Sample DNN Time 0.01173	Train Loss 0.0146	
+Epoch: [10][883/1713]	Per Sample Total Time 0.01204	Per Sample Data Time 0.00024	Per Sample DNN Time 0.01180	Train Loss 0.0146	
+Epoch: [10][983/1713]	Per Sample Total Time 0.01197	Per Sample Data Time 0.00022	Per Sample DNN Time 0.01175	Train Loss 0.0146	
+Epoch: [10][1083/1713]	Per Sample Total Time 0.01179	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01158	Train Loss 0.0146	
+Epoch: [10][1183/1713]	Per Sample Total Time 0.01184	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01165	Train Loss 0.0147	
+Epoch: [10][1283/1713]	Per Sample Total Time 0.01190	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01172	Train Loss 0.0147	
+Epoch: [10][1383/1713]	Per Sample Total Time 0.01194	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01177	Train Loss 0.0147	
+Epoch: [10][1483/1713]	Per Sample Total Time 0.01189	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01173	Train Loss 0.0147	
+Epoch: [10][1583/1713]	Per Sample Total Time 0.01176	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01161	Train Loss 0.0147	
+Epoch: [10][1683/1713]	Per Sample Total Time 0.01179	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01164	Train Loss 0.0147	
+start validation
+mAP: 0.301993
+AUC: 0.955118
+Avg Precision: 0.056732
+Avg Recall: 0.836964
+d_prime: 2.399414
+train_loss: 0.014710
+valid_loss: 0.694157
+validation finished
+Epoch-10 lr: 2.5e-05
+epoch 10 training time: 328.114
+---------------
+2025-08-27 03:35:35.835882
+current #epochs=11, #steps=17130
+Epoch: [11][70/1713]	Per Sample Total Time 0.01434	Per Sample Data Time 0.00201	Per Sample DNN Time 0.01234	Train Loss 0.0134	
+Epoch: [11][170/1713]	Per Sample Total Time 0.01221	Per Sample Data Time 0.00086	Per Sample DNN Time 0.01135	Train Loss 0.0134	
+Epoch: [11][270/1713]	Per Sample Total Time 0.01147	Per Sample Data Time 0.00055	Per Sample DNN Time 0.01092	Train Loss 0.0135	
+Epoch: [11][370/1713]	Per Sample Total Time 0.01179	Per Sample Data Time 0.00042	Per Sample DNN Time 0.01138	Train Loss 0.0135	
+Epoch: [11][470/1713]	Per Sample Total Time 0.01185	Per Sample Data Time 0.00034	Per Sample DNN Time 0.01151	Train Loss 0.0135	
+Epoch: [11][570/1713]	Per Sample Total Time 0.01184	Per Sample Data Time 0.00029	Per Sample DNN Time 0.01156	Train Loss 0.0136	
+Epoch: [11][670/1713]	Per Sample Total Time 0.01180	Per Sample Data Time 0.00025	Per Sample DNN Time 0.01155	Train Loss 0.0135	
+Epoch: [11][770/1713]	Per Sample Total Time 0.01161	Per Sample Data Time 0.00022	Per Sample DNN Time 0.01138	Train Loss 0.0135	
+Epoch: [11][870/1713]	Per Sample Total Time 0.01172	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01152	Train Loss 0.0135	
+Epoch: [11][970/1713]	Per Sample Total Time 0.01180	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01161	Train Loss 0.0136	
+Epoch: [11][1070/1713]	Per Sample Total Time 0.01184	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01167	Train Loss 0.0136	
+Epoch: [11][1170/1713]	Per Sample Total Time 0.01190	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01173	Train Loss 0.0136	
+Epoch: [11][1270/1713]	Per Sample Total Time 0.01193	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01178	Train Loss 0.0136	
+Epoch: [11][1370/1713]	Per Sample Total Time 0.01198	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01183	Train Loss 0.0135	
+Epoch: [11][1470/1713]	Per Sample Total Time 0.01200	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01187	Train Loss 0.0135	
+Epoch: [11][1570/1713]	Per Sample Total Time 0.01200	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01187	Train Loss 0.0135	
+Epoch: [11][1670/1713]	Per Sample Total Time 0.01201	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01189	Train Loss 0.0135	
+start validation
+mAP: 0.321699
+AUC: 0.957357
+Avg Precision: 0.058758
+Avg Recall: 0.849520
+d_prime: 2.433592
+train_loss: 0.013501
+valid_loss: 0.694092
+validation finished
+Epoch-11 lr: 2.5e-05
+epoch 11 training time: 334.541
+---------------
+2025-08-27 03:41:10.375904
+current #epochs=12, #steps=18843
+Epoch: [12][57/1713]	Per Sample Total Time 0.01455	Per Sample Data Time 0.00255	Per Sample DNN Time 0.01201	Train Loss 0.0128	
+Epoch: [12][157/1713]	Per Sample Total Time 0.01312	Per Sample Data Time 0.00097	Per Sample DNN Time 0.01215	Train Loss 0.0127	
+Epoch: [12][257/1713]	Per Sample Total Time 0.01283	Per Sample Data Time 0.00061	Per Sample DNN Time 0.01222	Train Loss 0.0126	
+Epoch: [12][357/1713]	Per Sample Total Time 0.01268	Per Sample Data Time 0.00045	Per Sample DNN Time 0.01223	Train Loss 0.0126	
+Epoch: [12][457/1713]	Per Sample Total Time 0.01254	Per Sample Data Time 0.00036	Per Sample DNN Time 0.01218	Train Loss 0.0126	
+Epoch: [12][557/1713]	Per Sample Total Time 0.01240	Per Sample Data Time 0.00031	Per Sample DNN Time 0.01209	Train Loss 0.0127	
+Epoch: [12][657/1713]	Per Sample Total Time 0.01228	Per Sample Data Time 0.00027	Per Sample DNN Time 0.01201	Train Loss 0.0127	
+Epoch: [12][757/1713]	Per Sample Total Time 0.01226	Per Sample Data Time 0.00024	Per Sample DNN Time 0.01203	Train Loss 0.0127	
+Epoch: [12][857/1713]	Per Sample Total Time 0.01225	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01204	Train Loss 0.0127	
+Epoch: [12][957/1713]	Per Sample Total Time 0.01229	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01209	Train Loss 0.0127	
+Epoch: [12][1057/1713]	Per Sample Total Time 0.01232	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01214	Train Loss 0.0127	
+Epoch: [12][1157/1713]	Per Sample Total Time 0.01231	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01214	Train Loss 0.0127	
+Epoch: [12][1257/1713]	Per Sample Total Time 0.01232	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01216	Train Loss 0.0128	
+Epoch: [12][1357/1713]	Per Sample Total Time 0.01234	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01219	Train Loss 0.0128	
+Epoch: [12][1457/1713]	Per Sample Total Time 0.01235	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01221	Train Loss 0.0127	
+Epoch: [12][1557/1713]	Per Sample Total Time 0.01236	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01222	Train Loss 0.0128	
+Epoch: [12][1657/1713]	Per Sample Total Time 0.01237	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01223	Train Loss 0.0128	
+start validation
+mAP: 0.323794
+AUC: 0.956825
+Avg Precision: 0.056271
+Avg Recall: 0.847217
+d_prime: 2.425336
+train_loss: 0.012771
+valid_loss: 0.694024
+validation finished
+Epoch-12 lr: 2.5e-05
+epoch 12 training time: 341.729
+---------------
+2025-08-27 03:46:52.104774
+current #epochs=13, #steps=20556
+Epoch: [13][44/1713]	Per Sample Total Time 0.01595	Per Sample Data Time 0.00302	Per Sample DNN Time 0.01293	Train Loss 0.0122	
+Epoch: [13][144/1713]	Per Sample Total Time 0.01372	Per Sample Data Time 0.00097	Per Sample DNN Time 0.01275	Train Loss 0.0125	
+Epoch: [13][244/1713]	Per Sample Total Time 0.01330	Per Sample Data Time 0.00060	Per Sample DNN Time 0.01270	Train Loss 0.0125	
+Epoch: [13][344/1713]	Per Sample Total Time 0.01301	Per Sample Data Time 0.00044	Per Sample DNN Time 0.01257	Train Loss 0.0124	
+Epoch: [13][444/1713]	Per Sample Total Time 0.01288	Per Sample Data Time 0.00035	Per Sample DNN Time 0.01254	Train Loss 0.0124	
+Epoch: [13][544/1713]	Per Sample Total Time 0.01282	Per Sample Data Time 0.00029	Per Sample DNN Time 0.01253	Train Loss 0.0123	
+Epoch: [13][644/1713]	Per Sample Total Time 0.01277	Per Sample Data Time 0.00025	Per Sample DNN Time 0.01252	Train Loss 0.0123	
+Epoch: [13][744/1713]	Per Sample Total Time 0.01275	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01252	Train Loss 0.0123	
+Epoch: [13][844/1713]	Per Sample Total Time 0.01276	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01256	Train Loss 0.0124	
+Epoch: [13][944/1713]	Per Sample Total Time 0.01273	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01254	Train Loss 0.0124	
+Epoch: [13][1044/1713]	Per Sample Total Time 0.01272	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01255	Train Loss 0.0124	
+Epoch: [13][1144/1713]	Per Sample Total Time 0.01273	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01257	Train Loss 0.0124	
+Epoch: [13][1244/1713]	Per Sample Total Time 0.01280	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01265	Train Loss 0.0124	
+Epoch: [13][1344/1713]	Per Sample Total Time 0.01280	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01265	Train Loss 0.0124	
+Epoch: [13][1444/1713]	Per Sample Total Time 0.01279	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01265	Train Loss 0.0124	
+Epoch: [13][1544/1713]	Per Sample Total Time 0.01276	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01263	Train Loss 0.0124	
+Epoch: [13][1644/1713]	Per Sample Total Time 0.01275	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01262	Train Loss 0.0124	
+start validation
+mAP: 0.323879
+AUC: 0.956006
+Avg Precision: 0.057743
+Avg Recall: 0.841438
+d_prime: 2.412806
+train_loss: 0.012389
+valid_loss: 0.693967
+validation finished
+Epoch-13 lr: 2.5e-05
+epoch 13 training time: 348.570
+---------------
+2025-08-27 03:52:40.675110
+current #epochs=14, #steps=22269
+Epoch: [14][31/1713]	Per Sample Total Time 0.01726	Per Sample Data Time 0.00422	Per Sample DNN Time 0.01303	Train Loss 0.0124	
+Epoch: [14][131/1713]	Per Sample Total Time 0.01382	Per Sample Data Time 0.00106	Per Sample DNN Time 0.01276	Train Loss 0.0120	
+Epoch: [14][231/1713]	Per Sample Total Time 0.01358	Per Sample Data Time 0.00062	Per Sample DNN Time 0.01296	Train Loss 0.0119	
+Epoch: [14][331/1713]	Per Sample Total Time 0.01321	Per Sample Data Time 0.00045	Per Sample DNN Time 0.01276	Train Loss 0.0119	
+Epoch: [14][431/1713]	Per Sample Total Time 0.01297	Per Sample Data Time 0.00036	Per Sample DNN Time 0.01261	Train Loss 0.0119	
+Epoch: [14][531/1713]	Per Sample Total Time 0.01283	Per Sample Data Time 0.00030	Per Sample DNN Time 0.01253	Train Loss 0.0119	
+Epoch: [14][631/1713]	Per Sample Total Time 0.01281	Per Sample Data Time 0.00026	Per Sample DNN Time 0.01255	Train Loss 0.0119	
+Epoch: [14][731/1713]	Per Sample Total Time 0.01279	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01257	Train Loss 0.0119	
+Epoch: [14][831/1713]	Per Sample Total Time 0.01267	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01247	Train Loss 0.0119	
+Epoch: [14][931/1713]	Per Sample Total Time 0.01262	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01244	Train Loss 0.0119	
+Epoch: [14][1031/1713]	Per Sample Total Time 0.01256	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01239	Train Loss 0.0119	
+Epoch: [14][1131/1713]	Per Sample Total Time 0.01251	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01235	Train Loss 0.0119	
+Epoch: [14][1231/1713]	Per Sample Total Time 0.01249	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01234	Train Loss 0.0119	
+Epoch: [14][1331/1713]	Per Sample Total Time 0.01246	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01231	Train Loss 0.0119	
+Epoch: [14][1431/1713]	Per Sample Total Time 0.01243	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01229	Train Loss 0.0119	
+Epoch: [14][1531/1713]	Per Sample Total Time 0.01238	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01225	Train Loss 0.0119	
+Epoch: [14][1631/1713]	Per Sample Total Time 0.01237	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01225	Train Loss 0.0119	
+start validation
+mAP: 0.320660
+AUC: 0.954093
+Avg Precision: 0.058404
+Avg Recall: 0.837463
+d_prime: 2.384228
+train_loss: 0.011937
+valid_loss: 0.693958
+validation finished
+Epoch-14 lr: 2.5e-05
+epoch 14 training time: 341.352
+---------------
+2025-08-27 03:58:22.027047
+current #epochs=15, #steps=23982
+Epoch: [15][18/1713]	Per Sample Total Time 0.01914	Per Sample Data Time 0.00752	Per Sample DNN Time 0.01162	Train Loss 0.0115	
+Epoch: [15][118/1713]	Per Sample Total Time 0.01358	Per Sample Data Time 0.00124	Per Sample DNN Time 0.01234	Train Loss 0.0115	
+Epoch: [15][218/1713]	Per Sample Total Time 0.01286	Per Sample Data Time 0.00070	Per Sample DNN Time 0.01217	Train Loss 0.0117	
+Epoch: [15][318/1713]	Per Sample Total Time 0.01255	Per Sample Data Time 0.00049	Per Sample DNN Time 0.01206	Train Loss 0.0117	
+Epoch: [15][418/1713]	Per Sample Total Time 0.01241	Per Sample Data Time 0.00038	Per Sample DNN Time 0.01202	Train Loss 0.0117	
+Epoch: [15][518/1713]	Per Sample Total Time 0.01239	Per Sample Data Time 0.00032	Per Sample DNN Time 0.01207	Train Loss 0.0116	
+Epoch: [15][618/1713]	Per Sample Total Time 0.01241	Per Sample Data Time 0.00027	Per Sample DNN Time 0.01214	Train Loss 0.0117	
+Epoch: [15][718/1713]	Per Sample Total Time 0.01231	Per Sample Data Time 0.00024	Per Sample DNN Time 0.01207	Train Loss 0.0117	
+Epoch: [15][818/1713]	Per Sample Total Time 0.01219	Per Sample Data Time 0.00022	Per Sample DNN Time 0.01197	Train Loss 0.0116	
+Epoch: [15][918/1713]	Per Sample Total Time 0.01203	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01183	Train Loss 0.0116	
+Epoch: [15][1018/1713]	Per Sample Total Time 0.01198	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01180	Train Loss 0.0116	
+Epoch: [15][1118/1713]	Per Sample Total Time 0.01200	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01183	Train Loss 0.0116	
+Epoch: [15][1218/1713]	Per Sample Total Time 0.01193	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01178	Train Loss 0.0116	
+Epoch: [15][1318/1713]	Per Sample Total Time 0.01188	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01173	Train Loss 0.0116	
+Epoch: [15][1418/1713]	Per Sample Total Time 0.01186	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01172	Train Loss 0.0116	
+Epoch: [15][1518/1713]	Per Sample Total Time 0.01187	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01173	Train Loss 0.0116	
+Epoch: [15][1618/1713]	Per Sample Total Time 0.01187	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01174	Train Loss 0.0116	
+start validation
+mAP: 0.324188
+AUC: 0.955051
+Avg Precision: 0.056682
+Avg Recall: 0.845943
+d_prime: 2.398415
+train_loss: 0.011597
+valid_loss: 0.693899
+validation finished
+Epoch-15 lr: 1.25e-05
+epoch 15 training time: 331.892
+---------------
+2025-08-27 04:03:53.919434
+current #epochs=16, #steps=25695
+Epoch: [16][5/1713]	Per Sample Total Time 0.04252	Per Sample Data Time 0.02888	Per Sample DNN Time 0.01365	Train Loss 0.0112	
+Epoch: [16][105/1713]	Per Sample Total Time 0.01470	Per Sample Data Time 0.00168	Per Sample DNN Time 0.01302	Train Loss 0.0111	
+Epoch: [16][205/1713]	Per Sample Total Time 0.01382	Per Sample Data Time 0.00089	Per Sample DNN Time 0.01293	Train Loss 0.0109	
+Epoch: [16][305/1713]	Per Sample Total Time 0.01352	Per Sample Data Time 0.00062	Per Sample DNN Time 0.01291	Train Loss 0.0109	
+Epoch: [16][405/1713]	Per Sample Total Time 0.01325	Per Sample Data Time 0.00048	Per Sample DNN Time 0.01278	Train Loss 0.0109	
+Epoch: [16][505/1713]	Per Sample Total Time 0.01320	Per Sample Data Time 0.00039	Per Sample DNN Time 0.01281	Train Loss 0.0109	
+Epoch: [16][605/1713]	Per Sample Total Time 0.01319	Per Sample Data Time 0.00034	Per Sample DNN Time 0.01286	Train Loss 0.0109	
+Epoch: [16][705/1713]	Per Sample Total Time 0.01316	Per Sample Data Time 0.00030	Per Sample DNN Time 0.01287	Train Loss 0.0109	
+Epoch: [16][805/1713]	Per Sample Total Time 0.01307	Per Sample Data Time 0.00026	Per Sample DNN Time 0.01280	Train Loss 0.0109	
+Epoch: [16][905/1713]	Per Sample Total Time 0.01300	Per Sample Data Time 0.00024	Per Sample DNN Time 0.01276	Train Loss 0.0109	
+Epoch: [16][1005/1713]	Per Sample Total Time 0.01295	Per Sample Data Time 0.00022	Per Sample DNN Time 0.01273	Train Loss 0.0109	
+Epoch: [16][1105/1713]	Per Sample Total Time 0.01290	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01269	Train Loss 0.0109	
+Epoch: [16][1205/1713]	Per Sample Total Time 0.01287	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01268	Train Loss 0.0109	
+Epoch: [16][1305/1713]	Per Sample Total Time 0.01283	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01264	Train Loss 0.0109	
+Epoch: [16][1405/1713]	Per Sample Total Time 0.01281	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01264	Train Loss 0.0109	
+Epoch: [16][1505/1713]	Per Sample Total Time 0.01278	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01262	Train Loss 0.0109	
+Epoch: [16][1605/1713]	Per Sample Total Time 0.01276	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01261	Train Loss 0.0109	
+Epoch: [16][1705/1713]	Per Sample Total Time 0.01273	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01258	Train Loss 0.0109	
+start validation
+mAP: 0.327409
+AUC: 0.953805
+Avg Precision: 0.058829
+Avg Recall: 0.839503
+d_prime: 2.380009
+train_loss: 0.010876
+valid_loss: 0.693826
+validation finished
+Epoch-16 lr: 1.25e-05
+epoch 16 training time: 347.769
+---------------
+2025-08-27 04:09:41.688588
+current #epochs=17, #steps=27408
+Epoch: [17][92/1713]	Per Sample Total Time 0.01398	Per Sample Data Time 0.00155	Per Sample DNN Time 0.01243	Train Loss 0.0107	
+Epoch: [17][192/1713]	Per Sample Total Time 0.01296	Per Sample Data Time 0.00077	Per Sample DNN Time 0.01219	Train Loss 0.0107	
+Epoch: [17][292/1713]	Per Sample Total Time 0.01261	Per Sample Data Time 0.00052	Per Sample DNN Time 0.01208	Train Loss 0.0105	
+Epoch: [17][392/1713]	Per Sample Total Time 0.01233	Per Sample Data Time 0.00040	Per Sample DNN Time 0.01193	Train Loss 0.0105	
+Epoch: [17][492/1713]	Per Sample Total Time 0.01210	Per Sample Data Time 0.00033	Per Sample DNN Time 0.01177	Train Loss 0.0106	
+Epoch: [17][592/1713]	Per Sample Total Time 0.01185	Per Sample Data Time 0.00028	Per Sample DNN Time 0.01158	Train Loss 0.0106	
+Epoch: [17][692/1713]	Per Sample Total Time 0.01191	Per Sample Data Time 0.00024	Per Sample DNN Time 0.01167	Train Loss 0.0106	
+Epoch: [17][792/1713]	Per Sample Total Time 0.01196	Per Sample Data Time 0.00022	Per Sample DNN Time 0.01174	Train Loss 0.0106	
+Epoch: [17][892/1713]	Per Sample Total Time 0.01201	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01181	Train Loss 0.0106	
+Epoch: [17][992/1713]	Per Sample Total Time 0.01206	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01188	Train Loss 0.0106	
+Epoch: [17][1092/1713]	Per Sample Total Time 0.01209	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01192	Train Loss 0.0106	
+Epoch: [17][1192/1713]	Per Sample Total Time 0.01212	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01196	Train Loss 0.0106	
+Epoch: [17][1292/1713]	Per Sample Total Time 0.01213	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01197	Train Loss 0.0106	
+Epoch: [17][1392/1713]	Per Sample Total Time 0.01214	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01200	Train Loss 0.0106	
+Epoch: [17][1492/1713]	Per Sample Total Time 0.01216	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01202	Train Loss 0.0106	
+Epoch: [17][1592/1713]	Per Sample Total Time 0.01215	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01202	Train Loss 0.0106	
+Epoch: [17][1692/1713]	Per Sample Total Time 0.01215	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01203	Train Loss 0.0106	
+start validation
+mAP: 0.326630
+AUC: 0.953156
+Avg Precision: 0.057865
+Avg Recall: 0.837258
+d_prime: 2.370586
+train_loss: 0.010578
+valid_loss: 0.693788
+validation finished
+Epoch-17 lr: 1.25e-05
+epoch 17 training time: 336.202
+---------------
+2025-08-27 04:15:17.890290
+current #epochs=18, #steps=29121
+Epoch: [18][79/1713]	Per Sample Total Time 0.01361	Per Sample Data Time 0.00190	Per Sample DNN Time 0.01172	Train Loss 0.0103	
+Epoch: [18][179/1713]	Per Sample Total Time 0.01298	Per Sample Data Time 0.00087	Per Sample DNN Time 0.01211	Train Loss 0.0104	
+Epoch: [18][279/1713]	Per Sample Total Time 0.01268	Per Sample Data Time 0.00058	Per Sample DNN Time 0.01211	Train Loss 0.0104	
+Epoch: [18][379/1713]	Per Sample Total Time 0.01250	Per Sample Data Time 0.00044	Per Sample DNN Time 0.01207	Train Loss 0.0104	
+Epoch: [18][479/1713]	Per Sample Total Time 0.01249	Per Sample Data Time 0.00035	Per Sample DNN Time 0.01213	Train Loss 0.0105	
+Epoch: [18][579/1713]	Per Sample Total Time 0.01249	Per Sample Data Time 0.00030	Per Sample DNN Time 0.01219	Train Loss 0.0105	
+Epoch: [18][679/1713]	Per Sample Total Time 0.01250	Per Sample Data Time 0.00026	Per Sample DNN Time 0.01223	Train Loss 0.0105	
+Epoch: [18][779/1713]	Per Sample Total Time 0.01246	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01222	Train Loss 0.0105	
+Epoch: [18][879/1713]	Per Sample Total Time 0.01245	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01224	Train Loss 0.0105	
+Epoch: [18][979/1713]	Per Sample Total Time 0.01252	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01233	Train Loss 0.0104	
+Epoch: [18][1079/1713]	Per Sample Total Time 0.01258	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01240	Train Loss 0.0104	
+Epoch: [18][1179/1713]	Per Sample Total Time 0.01259	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01242	Train Loss 0.0104	
+Epoch: [18][1279/1713]	Per Sample Total Time 0.01258	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01241	Train Loss 0.0104	
+Epoch: [18][1379/1713]	Per Sample Total Time 0.01256	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01240	Train Loss 0.0104	
+Epoch: [18][1479/1713]	Per Sample Total Time 0.01254	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01239	Train Loss 0.0104	
+Epoch: [18][1579/1713]	Per Sample Total Time 0.01252	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01238	Train Loss 0.0104	
+Epoch: [18][1679/1713]	Per Sample Total Time 0.01249	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01235	Train Loss 0.0104	
+start validation
+mAP: 0.327243
+AUC: 0.952374
+Avg Precision: 0.057596
+Avg Recall: 0.839843
+d_prime: 2.359356
+train_loss: 0.010416
+valid_loss: 0.693832
+validation finished
+Epoch-18 lr: 1.25e-05
+epoch 18 training time: 343.352
+---------------
+2025-08-27 04:21:01.242578
+current #epochs=19, #steps=30834
+Epoch: [19][66/1713]	Per Sample Total Time 0.01454	Per Sample Data Time 0.00188	Per Sample DNN Time 0.01266	Train Loss 0.0100	
+Epoch: [19][166/1713]	Per Sample Total Time 0.01352	Per Sample Data Time 0.00078	Per Sample DNN Time 0.01274	Train Loss 0.0101	
+Epoch: [19][266/1713]	Per Sample Total Time 0.01298	Per Sample Data Time 0.00051	Per Sample DNN Time 0.01247	Train Loss 0.0100	
+Epoch: [19][366/1713]	Per Sample Total Time 0.01277	Per Sample Data Time 0.00038	Per Sample DNN Time 0.01239	Train Loss 0.0100	
+Epoch: [19][466/1713]	Per Sample Total Time 0.01253	Per Sample Data Time 0.00031	Per Sample DNN Time 0.01222	Train Loss 0.0100	
+Epoch: [19][566/1713]	Per Sample Total Time 0.01230	Per Sample Data Time 0.00026	Per Sample DNN Time 0.01204	Train Loss 0.0100	
+Epoch: [19][666/1713]	Per Sample Total Time 0.01228	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01205	Train Loss 0.0101	
+Epoch: [19][766/1713]	Per Sample Total Time 0.01226	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01206	Train Loss 0.0101	
+Epoch: [19][866/1713]	Per Sample Total Time 0.01222	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01204	Train Loss 0.0101	
+Epoch: [19][966/1713]	Per Sample Total Time 0.01221	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01204	Train Loss 0.0101	
+Epoch: [19][1066/1713]	Per Sample Total Time 0.01220	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01204	Train Loss 0.0101	
+Epoch: [19][1166/1713]	Per Sample Total Time 0.01216	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01201	Train Loss 0.0101	
+Epoch: [19][1266/1713]	Per Sample Total Time 0.01211	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01197	Train Loss 0.0101	
+Epoch: [19][1366/1713]	Per Sample Total Time 0.01202	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01189	Train Loss 0.0101	
+Epoch: [19][1466/1713]	Per Sample Total Time 0.01205	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01192	Train Loss 0.0101	
+Epoch: [19][1566/1713]	Per Sample Total Time 0.01208	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01195	Train Loss 0.0101	
+Epoch: [19][1666/1713]	Per Sample Total Time 0.01209	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01198	Train Loss 0.0101	
+start validation
+mAP: 0.325581
+AUC: 0.951207
+Avg Precision: 0.057401
+Avg Recall: 0.836137
+d_prime: 2.342888
+train_loss: 0.010055
+valid_loss: 0.693805
+validation finished
+Epoch-19 lr: 1.25e-05
+epoch 19 training time: 334.613
+---------------
+2025-08-27 04:26:35.855950
+current #epochs=20, #steps=32547
+Epoch: [20][53/1713]	Per Sample Total Time 0.01465	Per Sample Data Time 0.00259	Per Sample DNN Time 0.01206	Train Loss 0.0100	
+Epoch: [20][153/1713]	Per Sample Total Time 0.01233	Per Sample Data Time 0.00094	Per Sample DNN Time 0.01139	Train Loss 0.0099	
+Epoch: [20][253/1713]	Per Sample Total Time 0.01216	Per Sample Data Time 0.00059	Per Sample DNN Time 0.01158	Train Loss 0.0097	
+Epoch: [20][353/1713]	Per Sample Total Time 0.01224	Per Sample Data Time 0.00043	Per Sample DNN Time 0.01181	Train Loss 0.0098	
+Epoch: [20][453/1713]	Per Sample Total Time 0.01226	Per Sample Data Time 0.00035	Per Sample DNN Time 0.01192	Train Loss 0.0098	
+Epoch: [20][553/1713]	Per Sample Total Time 0.01227	Per Sample Data Time 0.00029	Per Sample DNN Time 0.01198	Train Loss 0.0098	
+Epoch: [20][653/1713]	Per Sample Total Time 0.01226	Per Sample Data Time 0.00026	Per Sample DNN Time 0.01201	Train Loss 0.0098	
+Epoch: [20][753/1713]	Per Sample Total Time 0.01230	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01207	Train Loss 0.0099	
+Epoch: [20][853/1713]	Per Sample Total Time 0.01232	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01212	Train Loss 0.0099	
+Epoch: [20][953/1713]	Per Sample Total Time 0.01232	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01213	Train Loss 0.0099	
+Epoch: [20][1053/1713]	Per Sample Total Time 0.01229	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01212	Train Loss 0.0100	
+Epoch: [20][1153/1713]	Per Sample Total Time 0.01227	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01211	Train Loss 0.0100	
+Epoch: [20][1253/1713]	Per Sample Total Time 0.01230	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01214	Train Loss 0.0100	
+Epoch: [20][1353/1713]	Per Sample Total Time 0.01231	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01217	Train Loss 0.0100	
+Epoch: [20][1453/1713]	Per Sample Total Time 0.01232	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01218	Train Loss 0.0100	
+Epoch: [20][1553/1713]	Per Sample Total Time 0.01232	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01219	Train Loss 0.0100	
+Epoch: [20][1653/1713]	Per Sample Total Time 0.01232	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01219	Train Loss 0.0100	
+start validation
+mAP: 0.326032
+AUC: 0.951268
+Avg Precision: 0.059974
+Avg Recall: 0.828896
+d_prime: 2.343747
+train_loss: 0.009952
+valid_loss: 0.693693
+validation finished
+Epoch-20 lr: 6.25e-06
+epoch 20 training time: 338.954
+---------------
+2025-08-27 04:32:14.809724
+current #epochs=21, #steps=34260
+Epoch: [21][40/1713]	Per Sample Total Time 0.01581	Per Sample Data Time 0.00377	Per Sample DNN Time 0.01204	Train Loss 0.0094	
+Epoch: [21][140/1713]	Per Sample Total Time 0.01329	Per Sample Data Time 0.00113	Per Sample DNN Time 0.01216	Train Loss 0.0096	
+Epoch: [21][240/1713]	Per Sample Total Time 0.01279	Per Sample Data Time 0.00068	Per Sample DNN Time 0.01211	Train Loss 0.0098	
+Epoch: [21][340/1713]	Per Sample Total Time 0.01265	Per Sample Data Time 0.00049	Per Sample DNN Time 0.01215	Train Loss 0.0098	
+Epoch: [21][440/1713]	Per Sample Total Time 0.01259	Per Sample Data Time 0.00039	Per Sample DNN Time 0.01220	Train Loss 0.0098	
+Epoch: [21][540/1713]	Per Sample Total Time 0.01253	Per Sample Data Time 0.00033	Per Sample DNN Time 0.01220	Train Loss 0.0097	
+Epoch: [21][640/1713]	Per Sample Total Time 0.01244	Per Sample Data Time 0.00028	Per Sample DNN Time 0.01216	Train Loss 0.0097	
+Epoch: [21][740/1713]	Per Sample Total Time 0.01241	Per Sample Data Time 0.00025	Per Sample DNN Time 0.01215	Train Loss 0.0096	
+Epoch: [21][840/1713]	Per Sample Total Time 0.01237	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01214	Train Loss 0.0096	
+Epoch: [21][940/1713]	Per Sample Total Time 0.01236	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01215	Train Loss 0.0096	
+Epoch: [21][1040/1713]	Per Sample Total Time 0.01235	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01216	Train Loss 0.0096	
+Epoch: [21][1140/1713]	Per Sample Total Time 0.01236	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01218	Train Loss 0.0096	
+Epoch: [21][1240/1713]	Per Sample Total Time 0.01236	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01219	Train Loss 0.0096	
+Epoch: [21][1340/1713]	Per Sample Total Time 0.01240	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01224	Train Loss 0.0096	
+Epoch: [21][1440/1713]	Per Sample Total Time 0.01243	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01227	Train Loss 0.0096	
+Epoch: [21][1540/1713]	Per Sample Total Time 0.01244	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01229	Train Loss 0.0096	
+Epoch: [21][1640/1713]	Per Sample Total Time 0.01245	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01231	Train Loss 0.0096	
+start validation
+mAP: 0.324904
+AUC: 0.950095
+Avg Precision: 0.058785
+Avg Recall: 0.826536
+d_prime: 2.327477
+train_loss: 0.009605
+valid_loss: 0.693686
+validation finished
+Epoch-21 lr: 6.25e-06
+epoch 21 training time: 340.738
+---------------
+2025-08-27 04:37:55.547797
+current #epochs=22, #steps=35973
+Epoch: [22][27/1713]	Per Sample Total Time 0.01741	Per Sample Data Time 0.00499	Per Sample DNN Time 0.01242	Train Loss 0.0093	
+Epoch: [22][127/1713]	Per Sample Total Time 0.01313	Per Sample Data Time 0.00113	Per Sample DNN Time 0.01200	Train Loss 0.0092	
+Epoch: [22][227/1713]	Per Sample Total Time 0.01251	Per Sample Data Time 0.00065	Per Sample DNN Time 0.01186	Train Loss 0.0092	
+Epoch: [22][327/1713]	Per Sample Total Time 0.01226	Per Sample Data Time 0.00047	Per Sample DNN Time 0.01180	Train Loss 0.0092	
+Epoch: [22][427/1713]	Per Sample Total Time 0.01219	Per Sample Data Time 0.00037	Per Sample DNN Time 0.01182	Train Loss 0.0093	
+Epoch: [22][527/1713]	Per Sample Total Time 0.01214	Per Sample Data Time 0.00031	Per Sample DNN Time 0.01183	Train Loss 0.0093	
+Epoch: [22][627/1713]	Per Sample Total Time 0.01209	Per Sample Data Time 0.00026	Per Sample DNN Time 0.01182	Train Loss 0.0094	
+Epoch: [22][727/1713]	Per Sample Total Time 0.01208	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01185	Train Loss 0.0094	
+Epoch: [22][827/1713]	Per Sample Total Time 0.01209	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01188	Train Loss 0.0094	
+Epoch: [22][927/1713]	Per Sample Total Time 0.01209	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01190	Train Loss 0.0094	
+Epoch: [22][1027/1713]	Per Sample Total Time 0.01212	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01195	Train Loss 0.0094	
+Epoch: [22][1127/1713]	Per Sample Total Time 0.01209	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01193	Train Loss 0.0095	
+Epoch: [22][1227/1713]	Per Sample Total Time 0.01212	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01196	Train Loss 0.0095	
+Epoch: [22][1327/1713]	Per Sample Total Time 0.01213	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01199	Train Loss 0.0095	
+Epoch: [22][1427/1713]	Per Sample Total Time 0.01217	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01203	Train Loss 0.0095	
+Epoch: [22][1527/1713]	Per Sample Total Time 0.01219	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01206	Train Loss 0.0095	
+Epoch: [22][1627/1713]	Per Sample Total Time 0.01221	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01208	Train Loss 0.0095	
+start validation
+mAP: 0.325570
+AUC: 0.949932
+Avg Precision: 0.057976
+Avg Recall: 0.827917
+d_prime: 2.325238
+train_loss: 0.009480
+valid_loss: 0.693679
+validation finished
+Epoch-22 lr: 6.25e-06
+epoch 22 training time: 336.561
+---------------
+2025-08-27 04:43:32.108769
+current #epochs=23, #steps=37686
+Epoch: [23][14/1713]	Per Sample Total Time 0.02238	Per Sample Data Time 0.00949	Per Sample DNN Time 0.01290	Train Loss 0.0097	
+Epoch: [23][114/1713]	Per Sample Total Time 0.01363	Per Sample Data Time 0.00127	Per Sample DNN Time 0.01235	Train Loss 0.0094	
+Epoch: [23][214/1713]	Per Sample Total Time 0.01300	Per Sample Data Time 0.00070	Per Sample DNN Time 0.01230	Train Loss 0.0094	
+Epoch: [23][314/1713]	Per Sample Total Time 0.01281	Per Sample Data Time 0.00049	Per Sample DNN Time 0.01232	Train Loss 0.0094	
+Epoch: [23][414/1713]	Per Sample Total Time 0.01265	Per Sample Data Time 0.00039	Per Sample DNN Time 0.01227	Train Loss 0.0094	
+Epoch: [23][514/1713]	Per Sample Total Time 0.01262	Per Sample Data Time 0.00032	Per Sample DNN Time 0.01230	Train Loss 0.0094	
+Epoch: [23][614/1713]	Per Sample Total Time 0.01255	Per Sample Data Time 0.00027	Per Sample DNN Time 0.01227	Train Loss 0.0094	
+Epoch: [23][714/1713]	Per Sample Total Time 0.01251	Per Sample Data Time 0.00024	Per Sample DNN Time 0.01227	Train Loss 0.0094	
+Epoch: [23][814/1713]	Per Sample Total Time 0.01242	Per Sample Data Time 0.00022	Per Sample DNN Time 0.01221	Train Loss 0.0094	
+Epoch: [23][914/1713]	Per Sample Total Time 0.01240	Per Sample Data Time 0.00020	Per Sample DNN Time 0.01220	Train Loss 0.0093	
+Epoch: [23][1014/1713]	Per Sample Total Time 0.01235	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01217	Train Loss 0.0093	
+Epoch: [23][1114/1713]	Per Sample Total Time 0.01225	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01208	Train Loss 0.0094	
+Epoch: [23][1214/1713]	Per Sample Total Time 0.01221	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01205	Train Loss 0.0094	
+Epoch: [23][1314/1713]	Per Sample Total Time 0.01217	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01202	Train Loss 0.0094	
+Epoch: [23][1414/1713]	Per Sample Total Time 0.01214	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01200	Train Loss 0.0093	
+Epoch: [23][1514/1713]	Per Sample Total Time 0.01214	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01201	Train Loss 0.0093	
+Epoch: [23][1614/1713]	Per Sample Total Time 0.01211	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01198	Train Loss 0.0093	
+start validation
+mAP: 0.324305
+AUC: 0.949169
+Avg Precision: 0.055806
+Avg Recall: 0.830132
+d_prime: 2.314852
+train_loss: 0.009343
+valid_loss: 0.693706
+validation finished
+Epoch-23 lr: 6.25e-06
+epoch 23 training time: 334.655
+---------------
+2025-08-27 04:49:06.763788
+current #epochs=24, #steps=39399
+Epoch: [24][1/1713]	Per Sample Total Time 0.08944	Per Sample Data Time 0.07234	Per Sample DNN Time 0.01709	Train Loss 0.0076	
+Epoch: [24][101/1713]	Per Sample Total Time 0.01414	Per Sample Data Time 0.00146	Per Sample DNN Time 0.01268	Train Loss 0.0092	
+Epoch: [24][201/1713]	Per Sample Total Time 0.01317	Per Sample Data Time 0.00076	Per Sample DNN Time 0.01241	Train Loss 0.0092	
+Epoch: [24][301/1713]	Per Sample Total Time 0.01286	Per Sample Data Time 0.00052	Per Sample DNN Time 0.01233	Train Loss 0.0092	
+Epoch: [24][401/1713]	Per Sample Total Time 0.01280	Per Sample Data Time 0.00041	Per Sample DNN Time 0.01239	Train Loss 0.0092	
+Epoch: [24][501/1713]	Per Sample Total Time 0.01272	Per Sample Data Time 0.00033	Per Sample DNN Time 0.01238	Train Loss 0.0093	
+Epoch: [24][601/1713]	Per Sample Total Time 0.01260	Per Sample Data Time 0.00029	Per Sample DNN Time 0.01232	Train Loss 0.0093	
+Epoch: [24][701/1713]	Per Sample Total Time 0.01256	Per Sample Data Time 0.00025	Per Sample DNN Time 0.01230	Train Loss 0.0093	
+Epoch: [24][801/1713]	Per Sample Total Time 0.01253	Per Sample Data Time 0.00023	Per Sample DNN Time 0.01231	Train Loss 0.0093	
+Epoch: [24][901/1713]	Per Sample Total Time 0.01248	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01228	Train Loss 0.0093	
+Epoch: [24][1001/1713]	Per Sample Total Time 0.01248	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01229	Train Loss 0.0093	
+Epoch: [24][1101/1713]	Per Sample Total Time 0.01247	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01230	Train Loss 0.0093	
+Epoch: [24][1201/1713]	Per Sample Total Time 0.01243	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01226	Train Loss 0.0094	
+Epoch: [24][1301/1713]	Per Sample Total Time 0.01242	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01227	Train Loss 0.0094	
+Epoch: [24][1401/1713]	Per Sample Total Time 0.01244	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01229	Train Loss 0.0094	
+Epoch: [24][1501/1713]	Per Sample Total Time 0.01240	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01226	Train Loss 0.0094	
+Epoch: [24][1601/1713]	Per Sample Total Time 0.01240	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01227	Train Loss 0.0094	
+Epoch: [24][1701/1713]	Per Sample Total Time 0.01238	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01225	Train Loss 0.0094	
+start validation
+mAP: 0.325054
+AUC: 0.948969
+Avg Precision: 0.058949
+Avg Recall: 0.826246
+d_prime: 2.312149
+train_loss: 0.009365
+valid_loss: 0.693633
+validation finished
+Epoch-24 lr: 6.25e-06
+epoch 24 training time: 343.091
+---------------
+2025-08-27 04:54:49.854832
+current #epochs=25, #steps=41112
+Epoch: [25][88/1713]	Per Sample Total Time 0.01340	Per Sample Data Time 0.00154	Per Sample DNN Time 0.01186	Train Loss 0.0092	
+Epoch: [25][188/1713]	Per Sample Total Time 0.01266	Per Sample Data Time 0.00075	Per Sample DNN Time 0.01191	Train Loss 0.0092	
+Epoch: [25][288/1713]	Per Sample Total Time 0.01242	Per Sample Data Time 0.00050	Per Sample DNN Time 0.01192	Train Loss 0.0091	
+Epoch: [25][388/1713]	Per Sample Total Time 0.01243	Per Sample Data Time 0.00039	Per Sample DNN Time 0.01205	Train Loss 0.0092	
+Epoch: [25][488/1713]	Per Sample Total Time 0.01243	Per Sample Data Time 0.00032	Per Sample DNN Time 0.01212	Train Loss 0.0092	
+Epoch: [25][588/1713]	Per Sample Total Time 0.01245	Per Sample Data Time 0.00027	Per Sample DNN Time 0.01218	Train Loss 0.0091	
+Epoch: [25][688/1713]	Per Sample Total Time 0.01238	Per Sample Data Time 0.00024	Per Sample DNN Time 0.01215	Train Loss 0.0092	
+Epoch: [25][788/1713]	Per Sample Total Time 0.01235	Per Sample Data Time 0.00021	Per Sample DNN Time 0.01214	Train Loss 0.0092	
+Epoch: [25][888/1713]	Per Sample Total Time 0.01234	Per Sample Data Time 0.00019	Per Sample DNN Time 0.01215	Train Loss 0.0092	
+Epoch: [25][988/1713]	Per Sample Total Time 0.01231	Per Sample Data Time 0.00018	Per Sample DNN Time 0.01213	Train Loss 0.0092	
+Epoch: [25][1088/1713]	Per Sample Total Time 0.01225	Per Sample Data Time 0.00017	Per Sample DNN Time 0.01208	Train Loss 0.0092	
+Epoch: [25][1188/1713]	Per Sample Total Time 0.01219	Per Sample Data Time 0.00016	Per Sample DNN Time 0.01204	Train Loss 0.0092	
+Epoch: [25][1288/1713]	Per Sample Total Time 0.01221	Per Sample Data Time 0.00015	Per Sample DNN Time 0.01206	Train Loss 0.0092	
+Epoch: [25][1388/1713]	Per Sample Total Time 0.01222	Per Sample Data Time 0.00014	Per Sample DNN Time 0.01208	Train Loss 0.0092	
+Epoch: [25][1488/1713]	Per Sample Total Time 0.01221	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01208	Train Loss 0.0092	
+Epoch: [25][1588/1713]	Per Sample Total Time 0.01221	Per Sample Data Time 0.00013	Per Sample DNN Time 0.01208	Train Loss 0.0092	
+Epoch: [25][1688/1713]	Per Sample Total Time 0.01217	Per Sample Data Time 0.00012	Per Sample DNN Time 0.01205	Train Loss 0.0092	
+start validation
+mAP: 0.323983
+AUC: 0.948025
+Avg Precision: 0.060971
+Avg Recall: 0.820553
+d_prime: 2.299510
+train_loss: 0.009207
+valid_loss: 0.693624
+validation finished
+Epoch-25 lr: 3.125e-06
+epoch 25 training time: 338.680
+---------------Training Finished---------------
+weighted averaged model results
+mAP: 0.340667
+AUC: 0.959997
+Avg Precision: 0.058671
+Avg Recall: 0.859400
+d_prime: 2.475802
+train_loss: 0.000000
+valid_loss: 0.693624

ast_1_AS20k/ast_origin_implement/test-balanced-f10-t10-pTrue-b12-lr5e-5-decoupe/wa_result.csv

@@ -0,0 +1,5 @@
+3.406672498832421514e-01
+9.599974407170855928e-01
+5.867078735816431967e-02
+8.594002744825509632e-01
+2.475801985654465742e+00

pre_4_AS2M/conv_clap_1_2025-09-30_06-58-32/pretraining_AS2M.sh

@@ -0,0 +1,340 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_resolution="4, 8, 16"
+model_modalities_image_conv_in_chans="1, 256, 384, 768"
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_resolution="16,"
+    model_modalities_image_conv_in_chans="1, 768"
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_resolution=${model_modalities_image_conv_resolution} \
+    +model.modalities.image.conv_in_chans=${model_modalities_image_conv_in_chans} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_1_2025-09-30_06-59-40/pretraining_AS2M.sh

@@ -0,0 +1,339 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_resolution="4, 8, 16"
+model_modalities_image_conv_in_chans="1, 256, 384, 768"
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_resolution="16,"
+    model_modalities_image_conv_in_chans="1, 768"
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_resolution=${model_modalities_image_conv_resolution} \
+    +model.modalities.image.conv_in_chans=${model_modalities_image_conv_in_chans} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_1_2025-09-30_07-01-07/pretraining_AS2M.sh

@@ -0,0 +1,339 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_resolution='[4,8,16]'
+model_modalities_image_conv_in_chans='[1,256,384,768]'
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_resolution='[16]'
+    model_modalities_image_conv_in_chans='[1,768]'
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_resolution=${model_modalities_image_conv_resolution} \
+    +model.modalities.image.conv_in_chans=${model_modalities_image_conv_in_chans} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_1_2025-09-30_07-08-58/pretraining_AS2M.sh

@@ -0,0 +1,336 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_1_2025-09-30_07-14-17/pretraining_AS2M.sh

@@ -0,0 +1,336 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_1_2025-09-30_07-19-43/pretraining_AS2M.sh

@@ -0,0 +1,336 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_1_2025-09-30_07-25-52/pretraining_AS2M.sh

@@ -0,0 +1,336 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_1_2025-09-30_08-31-42/pretraining_AS2M.sh

@@ -0,0 +1,418 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=5
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=6
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_1_2025-09-30_08-31-59/pretraining_AS2M.sh

@@ -0,0 +1,416 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=5
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=6
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_2_2025-09-30_09-12-51/pretraining_AS2M.sh

@@ -0,0 +1,416 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=2
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=5
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=6
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_4_2025-09-30_07-37-48/pretraining_AS2M.sh

@@ -0,0 +1,387 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=4
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_4_2025-09-30_07-38-18/pretraining_AS2M.sh

@@ -0,0 +1,384 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=4
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_4_2025-09-30_07-42-31/pretraining_AS2M.sh

@@ -0,0 +1,384 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=4
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_4_2025-09-30_07-45-39/pretraining_AS2M.sh

@@ -0,0 +1,384 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=4
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_4_2025-09-30_07-49-28/pretraining_AS2M.sh

@@ -0,0 +1,384 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=4
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_4_2025-09-30_07-57-18/pretraining_AS2M.sh

@@ -0,0 +1,384 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=4
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_4_2025-09-30_08-05-21/pretraining_AS2M.sh

@@ -0,0 +1,384 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=4
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_4_2025-09-30_08-13-17/pretraining_AS2M.sh

@@ -0,0 +1,384 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=4
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/conv_clap_4_2025-09-30_08-23-09/pretraining_AS2M.sh

@@ -0,0 +1,384 @@
+#!/usr/bin/env bash
+# config options
+train_mode=conv_clap
+config_option=4
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+model_modalities_image_conv_option=0
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=1
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=2
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=3
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=12 # modify with model depth
+    model_add_conv=true
+    model_modalities_image_conv_option=4
+    model_depth=12 # 
+    checkpoint_keep_interval_updates=1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    +model.modalities.image.conv_option=${model_modalities_image_conv_option} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_0_2025-09-24_13-58-24/pretraining_AS2M.sh

@@ -0,0 +1,246 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=0
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_0_2025-09-24_14-09-31/pretraining_AS2M.sh

@@ -0,0 +1,246 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=0
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_0_2025-09-24_14-12-12/pretraining_AS2M.sh

@@ -0,0 +1,246 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=0
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_0_2025-09-24_14-17-47/pretraining_AS2M.sh

@@ -0,0 +1,246 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=0
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_1_2025-09-26_14-32-16/pretraining_AS2M.sh

@@ -0,0 +1,258 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_1_2025-09-26_14-33-34/pretraining_AS2M.sh

@@ -0,0 +1,258 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_1_2025-09-26_14-34-35/pretraining_AS2M.sh

@@ -0,0 +1,258 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_1_2025-09-26_14-39-04/pretraining_AS2M.sh

@@ -0,0 +1,258 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_1_2025-09-26_14-57-51/pretraining_AS2M.sh

@@ -0,0 +1,258 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=1
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_3_2025-09-27_05-57-32/pretraining_AS2M.sh

@@ -0,0 +1,282 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=3
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_4_2025-09-28_05-38-34/pretraining_AS2M.sh

@@ -0,0 +1,294 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=4
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_5_2025-09-28_06-51-25/pretraining_AS2M.sh

@@ -0,0 +1,306 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=5
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_5_2025-09-28_07-56-38/pretraining_AS2M.sh

@@ -0,0 +1,318 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=5
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_6_2025-09-28_08-28-48/pretraining_AS2M.sh

@@ -0,0 +1,318 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=6
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_6_2025-09-28_08-49-54/pretraining_AS2M.sh

@@ -0,0 +1,318 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=6
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_6_2025-09-28_08-55-19/pretraining_AS2M.sh

@@ -0,0 +1,318 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=6
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}

pre_4_AS2M/disp_6_2025-09-28_08-58-05/pretraining_AS2M.sh

@@ -0,0 +1,318 @@
+#!/usr/bin/env bash
+# config options
+train_mode=disp
+config_option=6
+# change world size
+
+# shared config
+SAVE_DIR_ROOT=/opt/gpfs/home/chushu/exp/eat/pre_4_AS2M
+checkpoint_save_dir=${SAVE_DIR_ROOT}/${train_mode}_${config_option}_$(date +"%Y-%m-%d_%H-%M-%S")
+checkpoint_restore_file=${checkpoint_save_dir}/checkpoint_last.pt
+
+# 脚本自身的绝对路径与文件名（解析符号链接）
+script_path="$(readlink -f -- "${BASH_SOURCE[0]}")"
+script_name="$(basename -- "$script_path")"
+# 创建目录并拷贝（保留权限与时间戳）
+mkdir -p -- "$checkpoint_save_dir"
+cp -p -- "$script_path" "$checkpoint_save_dir/$script_name"
+echo "script_path: ${script_path}"
+echo "checkpoint_save_dir: ${checkpoint_save_dir}"
+
+# default setting
+model_clone_batch=4
+dataset_batch_size=48
+model_clap_loss=0
+model_clap_loss_type="mse"  # option ce cosine l1
+model_clap_loss_layer=0
+average_top_k_layers=12
+model_add_conv=false
+model_depth=12
+model_dispersive_loss=0
+model_dispersive_loss_layer=0
+checkpoint_keep_interval_updates=1 # TODO change this parameter if need
+checkpoint_save_interval_updates=10000
+
+if [[ $train_mode == "default" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=0
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "disp" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=100.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=10000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=1
+    dataset_batch_size=384
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=0
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "disp" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M
+    task_load_clap_emb=false
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=null
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_dispersive_loss=1000.0
+    model_dispersive_loss_layer=10
+    checkpoint_keep_interval_updates=1
+elif [[ $train_mode == "clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_add_conv=false
+elif [[ $train_mode == "clap" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=1
+# loss type ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="ce"
+elif [[ $train_mode == "clap" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=48
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="l1"
+elif [[ $train_mode == "clap" && ${config_option} -eq 4 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="cosine"
+# loss layer ablation
+elif [[ $train_mode == "clap" && ${config_option} -eq 5 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=10
+elif [[ $train_mode == "clap" && ${config_option} -eq 6 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=8
+elif [[ $train_mode == "clap" && ${config_option} -eq 7 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=96
+    model_clap_loss=1.0
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    model_clap_loss_layer=6
+elif [[ $train_mode == "clap" && ${config_option} -eq 8 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=5.0
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "clap" && ${config_option} -eq 9 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    model_clap_loss=0.1
+    dataset_batch_size=96
+    average_top_k_layers=12
+    model_clap_loss_type="mse"
+    checkpoint_keep_interval_updates=-1
+elif [[ $train_mode == "ast" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=1.0
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 1 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.001
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 2 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_in
+    task_load_clap_emb=true
+    model_proj_type=4
+    model_clone_batch=4
+    model_clap_loss=0.01
+    dataset_batch_size=48
+elif [[ $train_mode == "ast" && ${config_option} -eq 3 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_AST/mlp_head_out
+    task_load_clap_emb=true
+    model_proj_type=6
+    model_clone_batch=4
+    dataset_batch_size=48
+elif [[ $train_mode == "conv_clap" && ${config_option} -eq 0 ]]; then
+    echo "Config ${train_mode} ${config_option}"
+    task_data=/opt/gpfs/home/chushu/data/audioset/setting/PRETRAIN_AS2M_w_CLAP
+    task_load_clap_emb=true
+    task_load_source_file=true
+    task_load_mel_file=false
+    model_proj_type=2
+    model_clone_batch=4
+    dataset_batch_size=64 # original 48 oom on 4090 24G change distributed_world_size
+    model_clap_loss=1.0
+    average_top_k_layers=11 # modify with model depth
+    model_add_conv=true
+    model_depth=11 # 
+    checkpoint_keep_interval_updates=-1 # default 1 
+    checkpoint_save_interval_updates=10000
+fi
+
+python fairseq_cli/hydra_train.py -m \
+    --config-dir ./EAT/config \
+    --config-name pretraining_AS2M \
+    common.user_dir=./EAT \
+    checkpoint.save_dir=${checkpoint_save_dir} \
+    checkpoint.restore_file=${checkpoint_restore_file} \
+    distributed_training.distributed_world_size=${1:-2} \
+    dataset.num_workers=24 \
+    dataset.data_buffer_size=48 \
+    dataset.batch_size=${dataset_batch_size} \
+    task.data=${task_data} \
+    task.h5_format=False \
+    task.load_clap_emb=${task_load_clap_emb} \
+    +task.load_source_file=${task_load_source_file} \
+    +task.load_mel_file=${task_load_mel_file} \
+    model.proj_type=${model_proj_type} \
+    model.clone_batch=${model_clone_batch} \
+    model.clap_loss=${model_clap_loss} \
+    model.average_top_k_layers=${average_top_k_layers} \
+    +model.add_conv=${model_add_conv} \
+    +model.clap_loss_type=${model_clap_loss_type} \
+    +model.clap_loss_layer=${model_clap_loss_layer} \
+    +model.dispersive_loss=${model_dispersive_loss} \
+    +model.dispersive_loss_layer=${model_dispersive_loss_layer} \
+    model.depth=${model_depth} \
+    checkpoint.keep_interval_updates=${checkpoint_keep_interval_updates} \
+    checkpoint.save_interval_updates=${checkpoint_save_interval_updates}